From 894f74fefb303be2ad18033cfb5e96530e80f51a Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Fri, 14 Nov 2025 15:57:53 +0100 Subject: [PATCH] add core-api controllers to maintain portainer, npm and organizr deploys --- STATUS.md | 46 ++- services/core-api/REFACTORING_PLAN.md | 77 +++-- services/core-api/src/config.py | 37 ++- .../controllers/infrastructure_controller.py | 266 +++++++++++++++++- services/core-api/src/main.py | 27 +- 5 files changed, 406 insertions(+), 47 deletions(-) diff --git a/STATUS.md b/STATUS.md index e2ddab3..7eb1af9 100644 --- a/STATUS.md +++ b/STATUS.md @@ -105,10 +105,11 @@ - [x] Create `/clients` directory structure - [x] Create `base.py` controller base class - [x] Add infrastructure settings to `config.py` (Portainer, NPM, Kuma URLs/credentials) + - [x] Create credentials management system (credentials.py gitignored, credentials.example.py template) + - [x] Update `main.py` routing to include infrastructure controller - [ ] Separate AI Orchestrator logic into `ai_controller.py` - [ ] Extract webscraper to `tools_controller.py` - [ ] Create `health_controller.py` for monitoring endpoints - - [ ] Update imports and routing in `main.py` - [x] **Infrastructure Management Controller:** Build automation API for service management - [x] Create `infrastructure_controller.py` with read/list endpoints - [x] **Portainer Integration:** HTTP client with access token authentication @@ -125,17 +126,21 @@ - [x] `create_proxy_host()` - Create new proxy configuration - [x] `get_certificates()` - List SSL certificates - [x] `create_certificate()` - Request Let's Encrypt cert - - [x] **Read/List Endpoints Implemented:** - - [x] `GET /infrastructure/health` - Check Portainer/NPM connectivity - - [x] `GET /infrastructure/services` - List all deployed services - - [x] `GET /infrastructure/services/{name}` - Get service details + - [x] **Read/List Endpoints Implemented & Tested:** + - [x] `GET /infrastructure/health` - Check Portainer/NPM connectivity ✅ TESTED + - [x] `GET /infrastructure/services` - List all deployed services ✅ TESTED (8 stacks) + - [x] `GET /infrastructure/services/{name}` - Get service details ✅ TESTED - [x] `GET /infrastructure/ports` - List allocated ports (skeleton) - - [x] `GET /infrastructure/domains` - List configured domains + - [x] `GET /infrastructure/domains` - List configured domains ✅ TESTED + - [x] **Portainer API Token:** Generated programmatically and configured + - [x] **Write Endpoints Implemented & Tested:** + - [x] `POST /infrastructure/services` - Deploy new service ✅ TESTED + - [x] `PUT /infrastructure/services/{name}` - Update service ✅ TESTED + - [x] `DELETE /infrastructure/services/{name}` - Remove service ✅ TESTED + - [x] `POST /infrastructure/proxy` - Create proxy host with SSL ✅ IMPLEMENTED - [ ] **Uptime Kuma Integration:** WebSocket client (deferred - complex Socket.IO) - - [ ] **Write Endpoints:** Deploy/update/delete operations - [ ] Replace ad-hoc shell scripts in `/stacks` with API endpoints - [ ] Add CLI wrapper for common operations - - [ ] Test endpoints with live infrastructure **Priority 2: AI Orchestrator Enhancement (Phase 2 - Memory Systems):** - [ ] Implement Tier 1: ConversationBufferMemory (in-memory, last 10 turns) @@ -276,6 +281,31 @@ None - All core services deployed and operational. ## Recent Updates +### 2025-11-14 Evening (Session 3) +**Core-API Refactoring - Write Endpoints:** +- ✅ Implemented POST /infrastructure/services - Deploy service from compose YAML +- ✅ Implemented PUT /infrastructure/services/{name} - Update service configuration +- ✅ Implemented DELETE /infrastructure/services/{name} - Remove service and stack +- ✅ Implemented POST /infrastructure/proxy - Create proxy host with optional SSL +- ✅ Tested all service management endpoints (POST/PUT/DELETE) with test-nginx stack +- ✅ Updated main.py API description with write endpoints + +**Status:** Infrastructure Management API complete (Phase 3 ✅) +**Next:** Refactor existing controllers (Phase 4) or begin AI Orchestrator Phase 2 + +### 2025-11-14 Evening (Session 2) +**Core-API Refactoring - Infrastructure Management:** +- ✅ Created credentials management system (credentials.py gitignored) +- ✅ Generated Portainer API token programmatically via API +- ✅ Integrated infrastructure controller into main.py +- ✅ Fixed Pydantic validation bug (status int→str conversion) +- ✅ Tested all infrastructure read endpoints with live data +- ✅ Verified 8 Portainer stacks detected +- ✅ Domains endpoint working with SSL status + +**Status:** Infrastructure read endpoints complete and tested +**Next:** Implement write endpoints (POST/PUT/DELETE) + ### 2025-11-13 Evening **Phase 1 Testing & Bug Fix:** - ✅ Completed comprehensive testing of Phase 1 implementation diff --git a/services/core-api/REFACTORING_PLAN.md b/services/core-api/REFACTORING_PLAN.md index 22b790b..3db144b 100644 --- a/services/core-api/REFACTORING_PLAN.md +++ b/services/core-api/REFACTORING_PLAN.md @@ -53,13 +53,13 @@ src/ ## Implementation Phases -### Phase 1: Infrastructure Setup 🔄 IN PROGRESS +### Phase 1: Infrastructure Setup ✅ COMPLETE - [x] Research API authentication methods - [x] Add infrastructure settings to config.py -- [ ] Create credentials.py for sensitive data (gitignored) -- [ ] Create credentials.example.py as template -- [ ] Update .gitignore to exclude credentials.py -- [ ] Update config.py to import from credentials module +- [x] Create credentials.py for sensitive data (gitignored) +- [x] Create credentials.example.py as template +- [x] Update .gitignore to exclude credentials.py +- [x] Update config.py to import from credentials module - [x] Create /controllers directory structure - [x] Create /clients directory structure - [x] Create base controller class @@ -70,17 +70,19 @@ src/ - [x] Add token storage/refresh mechanisms - [ ] Implement Uptime Kuma Socket.IO client (DEFERRED - WebSocket complexity) -### Phase 3: Infrastructure Controller 🔄 IN PROGRESS -- [x] GET /infrastructure/health - Check connectivity -- [x] GET /infrastructure/services - List all services -- [x] GET /infrastructure/services/{name} - Get service details +### Phase 3: Infrastructure Controller ✅ COMPLETE +- [x] GET /infrastructure/health - Check connectivity ✅ TESTED +- [x] GET /infrastructure/services - List all services ✅ TESTED +- [x] GET /infrastructure/services/{name} - Get service details ✅ TESTED - [x] GET /infrastructure/ports - List allocated ports (skeleton) -- [x] GET /infrastructure/domains - List configured domains -- [ ] POST /infrastructure/services - Deploy new service -- [ ] PUT /infrastructure/services/{name} - Update service -- [ ] DELETE /infrastructure/services/{name} - Remove service -- [ ] POST /infrastructure/monitoring/add - Auto-add Kuma monitor -- [ ] POST /infrastructure/proxy/add - Auto-add NPM proxy host +- [x] GET /infrastructure/domains - List configured domains ✅ TESTED +- [x] Integrate with main.py routing ✅ TESTED +- [x] Fix Pydantic validation issues (status field type conversion) +- [x] POST /infrastructure/services - Deploy new service ✅ TESTED +- [x] PUT /infrastructure/services/{name} - Update service ✅ TESTED +- [x] DELETE /infrastructure/services/{name} - Remove service ✅ TESTED +- [x] POST /infrastructure/proxy - Create NPM proxy host with optional SSL ✅ IMPLEMENTED +- [ ] POST /infrastructure/monitoring/add - Auto-add Kuma monitor (DEFERRED - Socket.IO complexity) ### Phase 4: Refactor Existing Controllers 📋 PENDING - [ ] Move AI endpoints to ai_controller.py @@ -115,15 +117,44 @@ src/ - `src/clients/npm_client.py` - `REFACTORING_PLAN.md` (this file) +### Session 2: Credentials & Testing (2025-11-14 Evening) +**Completed:** +- Created credentials management system (credentials.py gitignored, credentials.example.py committed) +- Updated config.py to import from credentials module with fallback +- Generated Portainer API token programmatically via API +- Integrated infrastructure controller into main.py +- Fixed Pydantic validation bug (status field int→str conversion) +- Tested all read endpoints with live Portainer/NPM infrastructure +- Verified 8 stacks detected, domains with SSL status working + +**Test Results:** +- ✅ GET /infrastructure/health - Portainer connected, NPM accessible +- ✅ GET /infrastructure/services - Returns 8 active stacks +- ✅ GET /infrastructure/services/{name} - Service lookup working +- ✅ GET /infrastructure/domains - Returns proxy hosts with SSL status +- ⚠️ NPM health check shows `false` (returns 302 redirect instead of 200) + +### Session 3: Write Endpoints (2025-11-14 Evening) +**Completed:** +- Created request/response models for write operations (DeployServiceRequest, UpdateServiceRequest, CreateProxyRequest, OperationResult) +- Implemented POST /infrastructure/services - Deploy new service from compose YAML +- Implemented PUT /infrastructure/services/{name} - Update existing service configuration +- Implemented DELETE /infrastructure/services/{name} - Remove service and stack +- Implemented POST /infrastructure/proxy - Create NPM proxy host with optional SSL certificate +- Updated main.py API description with write endpoints +- Tested all service management endpoints (POST/PUT/DELETE) with live Portainer instance + +**Test Results:** +- ✅ POST /infrastructure/services - Created test-nginx stack (ID: 30) +- ✅ PUT /infrastructure/services/test-nginx - Updated compose with environment variable +- ✅ DELETE /infrastructure/services/test-nginx - Removed stack successfully +- ✅ POST /infrastructure/proxy - Implemented (not tested to avoid production interference) + **Next Steps:** -1. Create credentials.py for secure credential management -2. Update config.py to import from credentials module -3. Add credentials.py to .gitignore -4. Update main.py to include infrastructure routes -5. Test endpoints with live infrastructure -6. Implement write/deploy operations -7. Refactor existing AI/tools/health endpoints -8. Create CLI wrappers +1. Refactor existing AI/tools/health endpoints into separate controllers (Phase 4) +2. Fix NPM health check to handle redirects +3. Implement port allocation detection logic +4. Create CLI wrappers for common operations ## API Authentication Strategy diff --git a/services/core-api/src/config.py b/services/core-api/src/config.py index d18f4b5..091a7e9 100644 --- a/services/core-api/src/config.py +++ b/services/core-api/src/config.py @@ -4,6 +4,25 @@ Global configuration for Core Code API from pydantic_settings import BaseSettings from functools import lru_cache +# Import infrastructure credentials from gitignored module +try: + from src.credentials import ( + PORTAINER_URL, PORTAINER_API_KEY, + NPM_URL, NPM_EMAIL, NPM_PASSWORD, + KUMA_URL, KUMA_USERNAME, KUMA_PASSWORD + ) +except ImportError: + # Fallback to empty strings if credentials.py doesn't exist + # (e.g., fresh clone before credentials setup) + PORTAINER_URL = "http://localhost:8001" + PORTAINER_API_KEY = "" + NPM_URL = "http://localhost:81" + NPM_EMAIL = "" + NPM_PASSWORD = "" + KUMA_URL = "http://localhost:3001" + KUMA_USERNAME = "" + KUMA_PASSWORD = "" + class Settings(BaseSettings): """Global application settings""" @@ -58,17 +77,17 @@ class Settings(BaseSettings): embedding_dimension: int = 384 embedding_batch_size: int = 32 - # Infrastructure Management - portainer_url: str = "http://localhost:8001" - portainer_api_key: str = "" + # Infrastructure Management (from credentials.py) + portainer_url: str = PORTAINER_URL + portainer_api_key: str = PORTAINER_API_KEY - npm_url: str = "http://localhost:81" - npm_email: str = "" - npm_password: str = "" + npm_url: str = NPM_URL + npm_email: str = NPM_EMAIL + npm_password: str = NPM_PASSWORD - kuma_url: str = "http://localhost:3001" - kuma_username: str = "" - kuma_password: str = "" + kuma_url: str = KUMA_URL + kuma_username: str = KUMA_USERNAME + kuma_password: str = KUMA_PASSWORD @property def model_aliases(self) -> dict: diff --git a/services/core-api/src/controllers/infrastructure_controller.py b/services/core-api/src/controllers/infrastructure_controller.py index 7aeb658..5eb55e4 100644 --- a/services/core-api/src/controllers/infrastructure_controller.py +++ b/services/core-api/src/controllers/infrastructure_controller.py @@ -5,8 +5,8 @@ Provides API endpoints for automated infrastructure management, including service deployment, configuration, and monitoring setup. """ from fastapi import APIRouter, HTTPException -from typing import List, Dict, Any, Optional -from pydantic import BaseModel +from typing import List, Dict, Any, Optional, Union +from pydantic import BaseModel, field_validator from src.controllers.base import BaseController from src.clients.portainer_client import get_portainer_client @@ -21,11 +21,20 @@ class ServiceInfo(BaseModel): """Information about a deployed service""" name: str stack_id: Optional[int] - status: str + status: Union[str, int] endpoint_id: Optional[int] ports: List[int] = [] domains: List[str] = [] + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert status to string representation""" + if isinstance(v, int): + # Portainer status: 1=active, 2=inactive + return "active" if v == 1 else "inactive" + return v + class PortInfo(BaseModel): """Information about an allocated port""" @@ -52,6 +61,41 @@ class InfrastructureHealth(BaseModel): total_proxy_hosts: int +# Request models for write operations +class DeployServiceRequest(BaseModel): + """Request to deploy a new service""" + name: str + compose_content: str + endpoint_id: int = 3 # Default to local endpoint + + +class UpdateServiceRequest(BaseModel): + """Request to update an existing service""" + compose_content: str + prune: bool = False + pull_image: bool = True + + +class CreateProxyRequest(BaseModel): + """Request to create a new proxy host""" + domain_names: List[str] + forward_host: str + forward_port: int + forward_scheme: str = "http" + ssl_enabled: bool = False + request_ssl_certificate: bool = False + block_exploits: bool = True + websocket_upgrade: bool = True + http2_support: bool = True + + +class OperationResult(BaseModel): + """Result of an infrastructure operation""" + success: bool + message: str + details: Optional[Dict[str, Any]] = None + + class InfrastructureController(BaseController): """ Controller for infrastructure management operations @@ -263,6 +307,222 @@ class InfrastructureController(BaseController): logger.error(f"Failed to list domains: {e}") raise HTTPException(status_code=500, detail=str(e)) + # Write endpoints + @router.post( + "/services", + response_model=OperationResult, + summary="Deploy a new service", + status_code=201 + ) + async def deploy_service(request: DeployServiceRequest): + """ + Deploy a new service via Portainer stack + + Args: + request: Service deployment configuration + + Returns: + Operation result with stack details + """ + portainer = get_portainer_client() + + try: + # Check if stack already exists + stacks = await portainer.get_stacks() + existing = next((s for s in stacks if s.get("Name") == request.name), None) + if existing: + raise HTTPException( + status_code=409, + detail=f"Service '{request.name}' already exists with ID {existing.get('Id')}" + ) + + # Create new stack + result = await portainer.create_stack( + name=request.name, + stack_file_content=request.compose_content, + endpoint_id=request.endpoint_id + ) + + logger.info(f"Deployed service '{request.name}' (stack ID: {result.get('Id')})") + + return OperationResult( + success=True, + message=f"Service '{request.name}' deployed successfully", + details=result + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to deploy service '{request.name}': {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.put( + "/services/{name}", + response_model=OperationResult, + summary="Update an existing service" + ) + async def update_service(name: str, request: UpdateServiceRequest): + """ + Update an existing service's compose configuration + + Args: + name: Service/stack name + request: Update configuration + + Returns: + Operation result with updated stack details + """ + portainer = get_portainer_client() + + try: + # Find stack by name + stacks = await portainer.get_stacks() + stack = next( + (s for s in stacks if s.get("Name", "").lower() == name.lower()), + None + ) + + if not stack: + raise HTTPException(status_code=404, detail=f"Service '{name}' not found") + + stack_id = stack.get("Id") + endpoint_id = stack.get("EndpointId") + + # Update stack + result = await portainer.update_stack( + stack_id=stack_id, + stack_file_content=request.compose_content, + endpoint_id=endpoint_id, + prune=request.prune, + pull_image=request.pull_image + ) + + logger.info(f"Updated service '{name}' (stack ID: {stack_id})") + + return OperationResult( + success=True, + message=f"Service '{name}' updated successfully", + details=result + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to update service '{name}': {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.delete( + "/services/{name}", + response_model=OperationResult, + summary="Delete a service" + ) + async def delete_service(name: str): + """ + Delete a service and remove its stack + + Args: + name: Service/stack name + + Returns: + Operation result confirmation + """ + portainer = get_portainer_client() + + try: + # Find stack by name + stacks = await portainer.get_stacks() + stack = next( + (s for s in stacks if s.get("Name", "").lower() == name.lower()), + None + ) + + if not stack: + raise HTTPException(status_code=404, detail=f"Service '{name}' not found") + + stack_id = stack.get("Id") + endpoint_id = stack.get("EndpointId") + + # Delete stack + await portainer.delete_stack( + stack_id=stack_id, + endpoint_id=endpoint_id + ) + + logger.info(f"Deleted service '{name}' (stack ID: {stack_id})") + + return OperationResult( + success=True, + message=f"Service '{name}' deleted successfully", + details={"stack_id": stack_id} + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to delete service '{name}': {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post( + "/proxy", + response_model=OperationResult, + summary="Create a new proxy host", + status_code=201 + ) + async def create_proxy(request: CreateProxyRequest): + """ + Create a new Nginx Proxy Manager proxy host + + Optionally request an SSL certificate from Let's Encrypt. + + Args: + request: Proxy host configuration + + Returns: + Operation result with proxy host details + """ + npm = get_npm_client() + + try: + certificate_id = 0 + + # Request SSL certificate if requested + if request.request_ssl_certificate: + logger.info(f"Requesting SSL certificate for {request.domain_names}") + cert_result = await npm.create_certificate( + domain_names=request.domain_names + ) + certificate_id = cert_result.get("id", 0) + logger.info(f"SSL certificate created: ID {certificate_id}") + + # Create proxy host + proxy_result = await npm.create_proxy_host( + domain_names=request.domain_names, + forward_host=request.forward_host, + forward_port=request.forward_port, + forward_scheme=request.forward_scheme, + certificate_id=certificate_id, + ssl_forced=request.ssl_enabled, + block_exploits=request.block_exploits, + websocket_upgrade=request.websocket_upgrade, + http2_support=request.http2_support + ) + + logger.info(f"Created proxy host for {request.domain_names} → {request.forward_host}:{request.forward_port}") + + return OperationResult( + success=True, + message=f"Proxy host created for {', '.join(request.domain_names)}", + details={ + "proxy_host": proxy_result, + "certificate_id": certificate_id if certificate_id > 0 else None + } + ) + + except Exception as e: + logger.error(f"Failed to create proxy host: {e}") + raise HTTPException(status_code=500, detail=str(e)) + return router diff --git a/services/core-api/src/main.py b/services/core-api/src/main.py index 1b64a0f..c72cea0 100644 --- a/services/core-api/src/main.py +++ b/services/core-api/src/main.py @@ -13,6 +13,7 @@ from src.api.v1.chat import router as chat_router from src.api.v1.models import router as models_router from src.api.v1.conversations import router as conversations_router from src.models.ollama_client import get_ollama_client, close_ollama_client +from src.controllers.infrastructure_controller import infrastructure_controller # Initialize settings settings = get_settings() @@ -79,6 +80,22 @@ app = FastAPI( - **Tier 1**: Fast in-memory buffer (last 10 turns) - **Tier 2/3**: Unified Qdrant storage (persistent + semantic search) + ### Infrastructure Management + **Read Endpoints:** + - `GET /infrastructure/health` - Check Portainer & NPM connectivity + - `GET /infrastructure/services` - List all deployed services + - `GET /infrastructure/services/{name}` - Get service details + - `GET /infrastructure/ports` - List allocated ports + - `GET /infrastructure/domains` - List configured domains + + **Write Endpoints:** + - `POST /infrastructure/services` - Deploy new service from compose YAML + - `PUT /infrastructure/services/{name}` - Update existing service + - `DELETE /infrastructure/services/{name}` - Remove service and stack + - `POST /infrastructure/proxy` - Create proxy host with optional SSL + + Automates infrastructure operations via Portainer and Nginx Proxy Manager APIs. + ### Web Scraper Intelligent web scraping with main content extraction. Perfect for extracting articles, documentation, and blog posts for LLM consumption. @@ -142,6 +159,7 @@ async def root(): "models": "/v1/models", "conversations": "/v1/conversations", "web_scraper": "/web-scraper/scrape", + "infrastructure": "/infrastructure", "health": "/health" } } @@ -171,10 +189,11 @@ async def health_check(): # Include routers -app.include_router(chat_router) # /v1/chat/completions -app.include_router(models_router) # /v1/models -app.include_router(conversations_router) # /v1/conversations -app.include_router(web_scraper_router) # /web-scraper/scrape +app.include_router(chat_router) # /v1/chat/completions +app.include_router(models_router) # /v1/models +app.include_router(conversations_router) # /v1/conversations +app.include_router(web_scraper_router) # /web-scraper/scrape +app.include_router(infrastructure_controller.create_router()) # /infrastructure/* # Global exception handler