commit 488a4e8a917cea848ca7af04743d97ab2e0b622a Author: Jeroen Schweitzer Date: Thu Dec 11 15:52:59 2025 +0100 Initial commit: core-api service extraction from portainer-core diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3db5eda --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# Core Code API Configuration + +# Application settings +APP_NAME="Core Code API" +APP_VERSION="1.0.0" +DEBUG=false + +# Server settings +HOST=0.0.0.0 +PORT=8083 + +# CORS settings (default allows all origins for internal use) +# CORS_ORIGINS=["http://192.168.86.149:82"] + +# Logging +LOG_LEVEL=INFO + +# Web Scraper Module +WEB_SCRAPER_REQUEST_TIMEOUT=30 +WEB_SCRAPER_MAX_REDIRECTS=5 +WEB_SCRAPER_USER_AGENT="Mozilla/5.0 (compatible; CoreCode/1.0)" +WEB_SCRAPER_DEFAULT_MAX_LENGTH=10000 +WEB_SCRAPER_MAX_LINKS_TO_EXTRACT=50 diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..27f17c6 --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,27 @@ +name: Build and Push + +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Login to Gitea Registry + uses: docker/login-action@v3 + with: + registry: git.schweitz.net + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: | + git.schweitz.net/jpmschweitzer/core-api:latest + git.schweitz.net/jpmschweitzer/core-api:${{ github.ref_name }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0980fb1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,139 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +coverage.json + +# Translations +*.mo +*.pot + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# IDE / Editor +.idea/ +.vscode/ +*.swp +*.swo +*~ +*.sublime-project +*.sublime-workspace + +# Logs +logs/ +*.log + +# Local data directories +task-data/ +data/ + +# Credentials and secrets +src/credentials.py +credentials.py +*.pem +*.key +secrets/ +.secrets + +# FastAPI / Uvicorn +.uvicorn/ + +# Database +*.db +*.sqlite +*.sqlite3 + +# OS files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Docker +.docker/ + +# Jupyter Notebook +.ipynb_checkpoints/ +*.ipynb + +# profiling data +.prof diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9f16702 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Install system dependencies (curl for healthcheck) +RUN apt-get update && apt-get install -y \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application +COPY src/ ./src/ +COPY static/ ./static/ + +ENV PYTHONPATH=/app + +EXPOSE 8083 + +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8083", "--workers", "1"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..215c827 --- /dev/null +++ b/README.md @@ -0,0 +1,218 @@ +# Core Code API + +OpenAPI-compatible functions for Open WebUI, providing web scraping and data processing capabilities. + +## Features + +### Web Scraper +- Intelligent content extraction using Trafilatura +- BeautifulSoup fallback for complex pages +- Configurable content length limits +- Optional link extraction +- Perfect for feeding webpage content to LLMs + +## Architecture + +``` +src/ +├── config.py # Global application settings +├── logging_config.py # Logging configuration +├── base_schema.py # Base Pydantic models +├── main.py # FastAPI application entry point +└── web_scraper/ # Web scraper module + ├── __init__.py + ├── config.py # Module-specific settings + ├── schemas.py # Pydantic request/response models + ├── service.py # Business logic + ├── router.py # API routes + └── exceptions.py # Custom exceptions +``` + +## Development + +### Requirements +- Python 3.12+ +- Docker (for containerized deployment) + +### Local Development + +```bash +# Install dependencies +pip install -r requirements.txt + +# Run locally +uvicorn src.main:app --reload --host 0.0.0.0 --port 8083 +``` + +### Adding New Dependencies + +**Important**: Dependencies use major version pinning (`~=`) for automatic patch updates while preventing breaking changes. + +1. Add package to `requirements.txt` with major version constraint: + ``` + package-name~=1.2.0 # Allows 1.2.x, blocks 1.3.0 + ``` + +2. Restart the container to install: + ```bash + docker restart core-api + ``` + +The container automatically runs `pip install -r requirements.txt` on every boot, so new dependencies are installed immediately on restart. + +**Version Pinning Best Practices**: +- Use `~=` (compatible release) for most packages: `fastapi~=0.115.0` +- Use `>=X,=0.3.17,<0.4.0` +- Allows automatic security patches without breaking changes +- Documented in PEP 440 + +### Docker Build + +```bash +# Build image +docker build -t core-code:latest . + +# Run container +docker run -p 8083:8083 core-code:latest +``` + +## Deployment + +### Portainer Stack + +1. Navigate to Portainer UI +2. Go to **Stacks** → **Add Stack** +3. Name: `core-code` +4. Upload `stacks/core-code.yml` or paste contents +5. Deploy + +### Environment Variables + +See `.env.example` for all available configuration options. + +## API Documentation + +Once deployed, access documentation at: +- **Swagger UI**: http://192.168.86.149:8083/docs +- **ReDoc**: http://192.168.86.149:8083/redoc +- **OpenAPI Spec**: http://192.168.86.149:8083/openapi.json + +## Integration with Open WebUI + +### Method 1: Functions (OpenAPI Import) +1. In Open WebUI, navigate to Functions +2. Import from OpenAPI spec: `http://192.168.86.149:8083/openapi.json` +3. Use functions directly in chat + +### Method 2: Pipelines +1. Create a pipeline that calls Core Code API endpoints +2. Use as data source for LLM workflows + +### Method 3: Direct API Calls +```python +import httpx + +async with httpx.AsyncClient() as client: + response = await client.post( + "http://192.168.86.149:8083/web-scraper/scrape", + json={ + "url": "https://example.com", + "extract_main_content": True + } + ) + data = response.json() +``` + +## API Endpoints + +### Web Scraper + +**POST /web-scraper/scrape** + +Scrape and extract content from a website. + +Request: +```json +{ + "url": "https://example.com/article", + "extract_main_content": true, + "include_links": false, + "max_length": 10000 +} +``` + +Response: +```json +{ + "url": "https://example.com/article", + "title": "Article Title", + "content": "Extracted article content...", + "extracted_at": "2025-11-12T19:30:00Z", + "content_length": 5432, + "links": null +} +``` + +## Logging + +Logs are written to: +- **Console**: stdout (captured by Docker) +- **File**: `/app/logs/app.log` (persisted via volume mount) + +Log format: +``` +2025-11-12 19:30:00 | INFO | src.web_scraper.service:scrape_url:45 | Starting scrape for URL: https://example.com +``` + +## Health Checks + +- **Endpoint**: `GET /health` +- **Docker**: Automatic health checks configured +- **Response**: `{"status": "healthy"}` + +## Security + +- Runs as non-root user (uid 1000) +- No authentication required (internal network only) +- CORS configured for same-network access +- Rate limiting: Not implemented (internal use only) + +## Future Modules + +The architecture supports adding new modules: +- Data transformation functions +- API integrations +- File processing +- Database queries + +Each module follows the same structure: +``` +src/ +└── module_name/ + ├── config.py + ├── schemas.py + ├── service.py + ├── router.py + └── exceptions.py +``` + +## Troubleshooting + +### Container won't start +```bash +docker logs core-code +``` + +### API not responding +```bash +curl http://192.168.86.149:8083/health +``` + +### Check OpenAPI spec +```bash +curl http://192.168.86.149:8083/openapi.json | jq +``` + +## License + +Internal use only. diff --git a/REQUESTED_SERVICES.md b/REQUESTED_SERVICES.md new file mode 100644 index 0000000..278a2a1 --- /dev/null +++ b/REQUESTED_SERVICES.md @@ -0,0 +1,446 @@ +# Requested Core-API Services for Core-AI Infrastructure Tools + +This document specifies the API endpoints needed by core-ai infrastructure tools. All requests from core-ai should go through core-api for centralized logging and access control. + +## Context + +The core-ai service is implementing 9 infrastructure tools in 3 logical clusters: +1. **Container Lifecycle** (4 tools) - containers.py +2. **Service Management** (3 tools) - services.py +3. **Monitoring & Resources** (2 tools) - monitoring.py + +These tools need corresponding core-api REST endpoints to perform operations via Portainer. + +--- + +## Cluster 1: Container Lifecycle Management + +### 1.1 List Containers + +**Endpoint:** `GET /v1/infrastructure/containers` + +**Query Parameters:** +- `status` (optional): Filter by status - "all", "running", "stopped", "paused" (default: "running") + +**Response:** +```json +[ + { + "Id": "abc123...", + "Names": ["/nginx"], + "State": "running", + "Status": "Up 3 days", + "Image": "nginx:latest", + "Ports": [ + {"PrivatePort": 80, "PublicPort": 8080, "Type": "tcp"}, + {"PrivatePort": 443, "PublicPort": 8443, "Type": "tcp"} + ], + "StartedAt": "2024-12-01T10:00:00Z" + } +] +``` + +**Implementation Notes:** +- Use `PortainerClient.list_containers(all_containers=True)` with Docker socket fallback +- Filter results based on `status` query parameter +- Return standard Docker API container list format + +--- + +### 1.2 Manage Container + +**Endpoint:** `POST /v1/infrastructure/containers/{container}/{action}` + +**Path Parameters:** +- `container`: Container name or ID (e.g., "nginx", "core-ai") +- `action`: One of: "start", "stop", "restart", "pause", "unpause", "remove" + +**Response (Success):** +```json +{ + "success": true, + "action": "restart", + "container": "nginx", + "message": "Container restarted successfully" +} +``` + +**Response (Error):** +```json +{ + "success": false, + "error": "Container not found", + "message": "Container 'nginx2' not found. Available containers: nginx, core-ai, ollama" +} +``` + +**Status Codes:** +- `200` - Success +- `304` - Not Modified (already in target state) +- `404` - Container not found +- `409` - Conflict (e.g., cannot remove running container) +- `500` - Server error + +**Implementation Notes:** +- For "restart": call stop then start +- For actions not yet in PortainerClient (pause, unpause, remove): + - Call Portainer API directly: `/api/endpoints/{endpoint_id}/docker/containers/{container_id}/{action}` +- Handle partial name matching (case-insensitive) +- Return helpful error messages suggesting `docker_list_containers()` when not found + +--- + +### 1.3 Inspect Container + +**Endpoint:** `GET /v1/infrastructure/containers/{container}` + +**Path Parameters:** +- `container`: Container name or ID + +**Query Parameters:** +- `details` (optional): Level of detail - "summary" (default), "full", "resources" + +**Response:** +```json +{ + "Id": "abc123...", + "Name": "/nginx", + "State": { + "Status": "running", + "Running": true, + "StartedAt": "2024-12-01T10:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "ExitCode": 0 + }, + "Config": { + "Image": "nginx:latest", + "Env": ["PATH=/usr/local/sbin:...", "NGINX_VERSION=1.25.0"], + "Cmd": ["nginx", "-g", "daemon off;"] + }, + "NetworkSettings": { + "Ports": { + "80/tcp": [{"HostIp": "0.0.0.0", "HostPort": "8080"}], + "443/tcp": [{"HostIp": "0.0.0.0", "HostPort": "8443"}] + }, + "Networks": { + "bridge": { + "IPAddress": "172.17.0.2", + "Gateway": "172.17.0.1" + } + } + }, + "HostConfig": { + "Memory": 536870912, + "NanoCpus": 1000000000, + "RestartPolicy": {"Name": "unless-stopped"} + }, + "Mounts": [ + { + "Type": "bind", + "Source": "/host/path", + "Destination": "/container/path" + } + ] +} +``` + +**Implementation Notes:** +- Use `PortainerClient.inspect_container(container)` which auto-detects endpoint and falls back to Docker socket +- Return full Docker inspect response +- The core-ai tool will handle formatting based on `details` level +- Return 404 if container not found + +--- + +### 1.4 Container Logs + +**Endpoint:** `GET /v1/infrastructure/containers/{container}/logs` + +**Path Parameters:** +- `container`: Container name or ID + +**Query Parameters:** +- `lines` (optional): Number of log lines (default: 50, max: 500) +- `since` (optional): Time filter - "1h", "30m", or ISO timestamp + +**Response:** +```json +{ + "container": "nginx", + "lines_requested": 50, + "since": null, + "logs": "2024-12-04T10:00:00.123Z Starting nginx...\n2024-12-04T10:00:01.456Z Ready to accept connections\n..." +} +``` + +**Implementation Notes:** +- Access Docker API directly: `GET /v1.41/containers/{container}/logs` + - Use Docker socket transport (httpx with uds) + - Parameters: `stdout=true`, `stderr=true`, `tail={lines}`, `timestamps=true` + - If `since` provided: add `since={unix_timestamp}` parameter +- Strip Docker stream headers (8-byte binary prefix per line) +- Return plain text logs with timestamps +- Return 404 if container not found + +--- + +## Cluster 2: Service Management + +### 2.1 List Services + +**Endpoint:** `GET /v1/infrastructure/services` (already exists, may need enhancement) + +**Query Parameters:** +- `stack` (optional): Filter by stack name + +**Response:** +```json +[ + { + "name": "portainer", + "stack_id": 1, + "status": "active", + "containers_running": 3, + "containers_total": 3, + "ports": [9000, 8000], + "domains": ["portainer.example.com"] + } +] +``` + +**Implementation Notes:** +- Enhance existing `/infrastructure/services` endpoint if needed +- Ensure it returns stack/service information from Portainer +- Include container counts (running/total) + +--- + +### 2.2 Manage Service + +**Endpoint:** `POST /v1/infrastructure/services/{service}/{action}` + +**Path Parameters:** +- `service`: Service/stack name +- `action`: One of: "start", "stop", "restart", "scale" + +**Request Body (for scale action):** +```json +{ + "replicas": 3 +} +``` + +**Response:** +```json +{ + "success": true, + "action": "restart", + "service": "web", + "message": "Service restarted successfully" +} +``` + +**Implementation Notes:** +- For "start"/"stop": Use Portainer stack start/stop API +- For "restart": Stop then start the stack +- For "scale": Update stack with new replica count + - This may require updating stack compose file + +--- + +### 2.3 Service Status + +**Endpoint:** `GET /v1/infrastructure/services/{service}/status` + +**Path Parameters:** +- `service`: Service/stack name + +**Response:** +```json +{ + "name": "web", + "status": "active", + "stack_id": 5, + "containers": [ + { + "name": "web_app_1", + "status": "running", + "health": "healthy", + "uptime": "2 days" + } + ], + "replica_status": "3/3 running", + "resources": { + "memory_total": "1.2 GB", + "cpu_usage": "15%" + }, + "recent_events": [ + {"time": "2024-12-04T09:00:00Z", "action": "container_start", "container": "web_app_3"} + ] +} +``` + +**Implementation Notes:** +- Get stack details from Portainer +- Get individual container statuses +- Calculate aggregate resource usage +- May require querying Docker events API for recent events + +--- + +## Cluster 3: Monitoring & Resources + +### 3.1 System Resources + +**Endpoint:** `GET /v1/infrastructure/resources/system` + +**Response:** +```json +{ + "cpu": { + "cores": 8, + "usage_percent": 45.2, + "load_average": [2.5, 2.3, 2.1] + }, + "memory": { + "total_bytes": 16777216000, + "used_bytes": 8388608000, + "available_bytes": 8388608000, + "usage_percent": 50.0 + }, + "disk": { + "total_bytes": 500000000000, + "used_bytes": 250000000000, + "available_bytes": 250000000000, + "usage_percent": 50.0 + }, + "network": { + "interfaces": { + "eth0": { + "rx_bytes": 1000000000, + "tx_bytes": 500000000 + } + } + } +} +``` + +**Implementation Notes:** +- Use Docker system info API: `GET /v1.41/system/df` +- May also use `GET /v1.41/info` for system-wide stats +- Calculate percentages and format nicely +- Include load averages from system stats + +--- + +### 3.2 Container Resources + +**Endpoint:** `GET /v1/infrastructure/resources/containers` + +**Query Parameters:** +- `container` (optional): Specific container name/ID (if omitted, return all) + +**Response:** +```json +[ + { + "name": "nginx", + "cpu_percent": 5.2, + "memory_usage_bytes": 45000000, + "memory_limit_bytes": 100000000, + "memory_percent": 45.0, + "network_rx_bytes": 50000000, + "network_tx_bytes": 25000000, + "block_read_bytes": 10000000, + "block_write_bytes": 5000000 + } +] +``` + +**Implementation Notes:** +- Use Docker stats API: `GET /v1.41/containers/{id}/stats?stream=false` +- If `container` param provided: return single container stats +- If omitted: return stats for all running containers +- Calculate percentages where applicable +- Stats API returns real-time metrics (one-time snapshot, not streaming) + +--- + +## Implementation Priority + +**Phase 1 (Needed immediately for core-ai):** +1. `GET /v1/infrastructure/containers` - List containers +2. `POST /v1/infrastructure/containers/{container}/{action}` - Manage containers +3. `GET /v1/infrastructure/containers/{container}` - Inspect container +4. `GET /v1/infrastructure/containers/{container}/logs` - Container logs + +**Phase 2 (Needed for full infrastructure tools):** +5. `POST /v1/infrastructure/services/{service}/{action}` - Manage services +6. `GET /v1/infrastructure/services/{service}/status` - Service status +7. `GET /v1/infrastructure/resources/system` - System resources +8. `GET /v1/infrastructure/resources/containers` - Container resources + +--- + +## Security & Access Control + +All endpoints should: +- Log all requests (especially write operations) +- Support OIDC authentication when enabled +- Require admin privileges for destructive operations (remove, scale) +- Rate limit to prevent abuse +- Validate input parameters +- Return sanitized errors (no sensitive data in error messages) + +--- + +## Error Handling + +Standard error response format: +```json +{ + "error": "ContainerNotFound", + "message": "Container 'nginx2' not found", + "details": { + "container": "nginx2", + "available_containers": ["nginx", "core-ai", "ollama"] + } +} +``` + +Common error codes: +- `400` - Bad Request (invalid parameters) +- `404` - Not Found (container/service doesn't exist) +- `409` - Conflict (invalid state transition) +- `500` - Internal Server Error (Portainer/Docker API failed) +- `503` - Service Unavailable (Portainer/Docker not accessible) + +--- + +## Testing + +Each endpoint should have: +- Unit tests (mock Portainer client) +- Integration tests (real Portainer/Docker) +- Error case tests (not found, permission denied, etc.) +- Performance tests (ensure response times < 2s) + +--- + +## Questions / Decisions Needed + +1. **Authentication**: Should container management require admin role, or allow read-only for all users? +2. **Rate Limiting**: What limits should be applied to prevent abuse? +3. **Caching**: Should container lists be cached? (TTL: 5s?) +4. **Async**: Should heavy operations (like logs) be async with job IDs? +5. **Webhooks**: Should operations emit events for monitoring? + +--- + +## Notes + +- All endpoints follow RESTful conventions +- Use existing PortainerClient methods where available +- Fall back to Docker socket when Portainer doesn't have data +- Log all operations with timestamps, user, and outcome +- Consider adding `/v1/infrastructure/containers/search` for fuzzy name matching diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6c3d88a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,27 @@ +# FastAPI and ASGI server +fastapi~=0.115.0 +uvicorn[standard]>=0.34.0 # Updated for google-adk compatibility +pydantic>=2.11.1,<3.0.0 # Required for google-cloud-aiplatform[agent-engines] +pydantic-settings>=2.10.1 + +# HTTP client +httpx>=0.28.0 # Required for google-adk +python-socketio[asyncio_client]~=5.11.0 + +# Web scraping +beautifulsoup4~=4.12.0 +trafilatura~=1.12.0 +lxml~=5.3.0 +duckduckgo-search~=4.1.0 + +# Utilities +python-multipart~=0.0.12 +python-dotenv~=1.0.0 +python-json-logger~=2.0.0 +pytz~=2024.1 +dnspython~=2.7.0 + +# Authentication & Security +PyJWT[crypto]~=2.9.0 +python-jose[cryptography]~=3.3.0 +cryptography~=43.0.0 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..17968bb --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,6 @@ +""" +Core Code API - OpenAPI-compatible functions for Open WebUI +""" + +__version__ = "1.0.0" +__author__ = "Core Code Team" diff --git a/src/auth/__init__.py b/src/auth/__init__.py new file mode 100644 index 0000000..dcee092 --- /dev/null +++ b/src/auth/__init__.py @@ -0,0 +1,5 @@ +""" +Authentication module for core-api + +Provides OIDC/OAuth2 authentication via Authentik +""" diff --git a/src/auth/oidc.py b/src/auth/oidc.py new file mode 100644 index 0000000..f381c13 --- /dev/null +++ b/src/auth/oidc.py @@ -0,0 +1,336 @@ +""" +OIDC Authentication Module + +Provides OAuth2/OIDC token validation for FastAPI using Authentik as IdP. +Implements bearer token authentication with JWT verification. +""" +from fastapi import Depends, HTTPException, Security, Request +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from jose import jwt, JWTError +import httpx +from functools import lru_cache +from typing import Dict, Optional +from src.logging_config import get_logger + +logger = get_logger(__name__) +security = HTTPBearer(auto_error=False) + + +class OIDCConfig: + """OIDC configuration from environment""" + + def __init__(self): + # These will be set from environment variables in config.py + self.enabled = False + self.issuer = "" + self.audience = "" + self.jwks_uri = "" + + def configure(self, enabled: bool, issuer: str, audience: str): + """Configure OIDC settings""" + self.enabled = enabled + self.issuer = issuer + self.audience = audience + self.jwks_uri = f"{issuer.rstrip('/')}/jwks/" + logger.info(f"OIDC configured: enabled={enabled}, issuer={issuer}") + + +# Global OIDC config instance +oidc_config = OIDCConfig() + + +@lru_cache(maxsize=1) +def get_jwks() -> Dict: + """ + Fetch JSON Web Key Set (JWKS) from Authentik + + Cached to avoid repeated requests. Cache is cleared on server restart. + + Returns: + JWKS dictionary containing public keys for token verification + + Raises: + HTTPException: If JWKS fetch fails + """ + if not oidc_config.enabled: + return {} + + try: + logger.debug(f"Fetching JWKS from {oidc_config.jwks_uri}") + response = httpx.get(oidc_config.jwks_uri, timeout=10.0) + response.raise_for_status() + jwks = response.json() + logger.info(f"JWKS fetched successfully ({len(jwks.get('keys', []))} keys)") + return jwks + except Exception as e: + logger.error(f"Failed to fetch JWKS: {e}") + raise HTTPException( + status_code=503, + detail="Authentication service unavailable" + ) + + +async def get_current_user( + credentials: Optional[HTTPAuthorizationCredentials] = Security(security) +) -> Optional[Dict]: + """ + Validate OIDC token from Authorization: Bearer header + + Extracts and validates JWT token from request header. Verifies: + - Token signature using JWKS + - Token expiration + - Issuer matches Authentik + - Audience matches core-api + + Args: + credentials: HTTP Bearer token from Authorization header + + Returns: + User claims dictionary containing email, name, groups, etc. + Returns None if OIDC is disabled (allows unauthenticated access) + + Raises: + HTTPException 401: If token is invalid, expired, or missing when OIDC enabled + """ + # If OIDC is disabled, allow all requests (no authentication) + if not oidc_config.enabled: + logger.debug("OIDC disabled - allowing unauthenticated access") + return None + + # OIDC enabled - token required + if not credentials: + logger.warning("Authentication required but no token provided") + raise HTTPException( + status_code=401, + detail="Authentication required", + headers={"WWW-Authenticate": "Bearer"}, + ) + + token = credentials.credentials + + try: + # Decode token header to get key ID + unverified_header = jwt.get_unverified_header(token) + kid = unverified_header.get("kid") + + if not kid: + raise HTTPException(status_code=401, detail="Invalid token format") + + # Find matching key in JWKS + jwks = get_jwks() + rsa_key = None + + for key in jwks.get("keys", []): + if key.get("kid") == kid: + rsa_key = key + break + + if not rsa_key: + logger.warning(f"No matching key found for kid: {kid}") + raise HTTPException(status_code=401, detail="Invalid token key") + + # Verify and decode token + payload = jwt.decode( + token, + rsa_key, + algorithms=["RS256"], + audience=oidc_config.audience, + issuer=oidc_config.issuer, + ) + + user_email = payload.get("email", "unknown") + logger.info(f"Authenticated user: {user_email}") + + return payload + + except jwt.ExpiredSignatureError: + logger.warning("Token expired") + raise HTTPException( + status_code=401, + detail="Token expired", + headers={"WWW-Authenticate": "Bearer"}, + ) + except jwt.JWTClaimsError as e: + logger.warning(f"Invalid token claims: {e}") + raise HTTPException( + status_code=401, + detail="Invalid token claims", + headers={"WWW-Authenticate": "Bearer"}, + ) + except JWTError as e: + logger.error(f"JWT validation error: {e}") + raise HTTPException( + status_code=401, + detail="Invalid authentication token", + headers={"WWW-Authenticate": "Bearer"}, + ) + except Exception as e: + logger.error(f"Unexpected authentication error: {e}") + raise HTTPException( + status_code=500, + detail="Authentication error", + ) + + +async def get_admin_user( + user: Optional[Dict] = Depends(get_current_user) +) -> Dict: + """ + Require admin group membership + + Use this dependency for endpoints that require admin access. + Checks if user is member of 'admin' group in Authentik. + + Args: + user: User claims from get_current_user + + Returns: + User claims dictionary if user is admin + + Raises: + HTTPException 403: If user is not in admin group + HTTPException 401: If OIDC enabled but user not authenticated + """ + # If OIDC disabled, allow all (backward compatibility) + if not oidc_config.enabled or user is None: + logger.debug("OIDC disabled - allowing admin access") + return {"email": "unauthenticated", "groups": ["admin"]} + + # Check admin group membership + groups = user.get("groups", []) + + if "admin" not in groups and "authentik Admins" not in groups: + user_email = user.get("email", "unknown") + logger.warning(f"User {user_email} attempted admin access (groups: {groups})") + raise HTTPException( + status_code=403, + detail="Admin access required" + ) + + return user + + +async def get_optional_user( + credentials: Optional[HTTPAuthorizationCredentials] = Security(security) +) -> Optional[Dict]: + """ + Optional authentication - allows both authenticated and unauthenticated access + + Use for endpoints that should be accessible to everyone but can provide + enhanced functionality for authenticated users. + + Args: + credentials: HTTP Bearer token from Authorization header + + Returns: + User claims if valid token provided, None otherwise + """ + if not credentials or not oidc_config.enabled: + return None + + try: + return await get_current_user(credentials) + except HTTPException: + # Invalid token - return None instead of raising + return None + + +async def get_forward_auth_user( + request: Request +) -> Optional[Dict]: + """ + Authentik Forward Auth authentication for external access via NPM + + This dependency allows: + - External access through api.schweitz.net (with Authentik forward auth headers) - REQUIRES authentication + - Internal direct access (no forward auth headers) - ALLOWED without authentication + + When accessing through NPM with Authentik forward auth enabled, NPM adds headers like: + - X-authentik-username + - X-authentik-email + - X-authentik-groups + - X-authentik-name + - X-authentik-uid + + Args: + request: FastAPI request object containing headers + + Returns: + User info dict if authenticated via forward auth headers + None if accessed internally (no forward auth headers) + + Raises: + HTTPException 401: If forward auth headers present but invalid/incomplete + """ + # Check for Authentik forward auth headers + username = request.headers.get("x-authentik-username") + email = request.headers.get("x-authentik-email") + groups = request.headers.get("x-authentik-groups") + name = request.headers.get("x-authentik-name") + uid = request.headers.get("x-authentik-uid") + + # If NO forward auth headers present, this is internal access - allow it + if not username and not email: + logger.debug("No forward auth headers - allowing internal access") + return None + + # Forward auth headers present (external access via api.schweitz.net) + # Validate authentication + if not username or not email: + logger.warning("Incomplete forward auth headers detected") + raise HTTPException( + status_code=401, + detail="Authentication required - incomplete forward auth headers" + ) + + # Parse groups (comma-separated string to list) + groups_list = [g.strip() for g in groups.split(",")] if groups else [] + + user_info = { + "username": username, + "email": email, + "name": name or username, + "groups": groups_list, + "uid": uid, + "auth_method": "forward_auth" + } + + logger.info(f"Authenticated via forward auth: {email} (groups: {groups_list})") + return user_info + + +async def get_forward_auth_admin( + user: Optional[Dict] = Depends(get_forward_auth_user) +) -> Dict: + """ + Require admin access for external requests, allow all internal requests + + Use this dependency for endpoints that require admin access when accessed + externally through api.schweitz.net, but allow unrestricted internal access. + + Args: + user: User info from get_forward_auth_user + + Returns: + User info dict if user is admin or if accessed internally + + Raises: + HTTPException 403: If external user is not in admin/authentik Admins group + """ + # Internal access (no forward auth headers) - allow all + if user is None: + logger.debug("Internal access - allowing without admin check") + return {"email": "internal", "groups": ["admin"], "auth_method": "internal"} + + # External access - check admin group membership + groups = user.get("groups", []) + + if "admin" not in groups and "authentik Admins" not in groups: + user_email = user.get("email", "unknown") + logger.warning(f"User {user_email} attempted admin access (groups: {groups})") + raise HTTPException( + status_code=403, + detail="Admin access required" + ) + + return user diff --git a/src/base_schema.py b/src/base_schema.py new file mode 100644 index 0000000..6d1d3e3 --- /dev/null +++ b/src/base_schema.py @@ -0,0 +1,45 @@ +""" +Base Pydantic models for consistent schema behavior +""" +from pydantic import BaseModel, ConfigDict +from datetime import datetime +from typing import Any + + +class BaseSchema(BaseModel): + """ + Base Pydantic model with standardized configuration + + All schemas should inherit from this to ensure consistent behavior: + - Consistent datetime serialization + - Strict validation by default + - JSON schema generation + """ + + model_config = ConfigDict( + # Strict type validation + strict=False, + + # Allow population by field name + populate_by_name=True, + + # Use enum values in JSON + use_enum_values=True, + + # Validate assignments after initialization + validate_assignment=True, + + # Serialize datetime to ISO format + json_encoders={ + datetime: lambda v: v.isoformat() if v else None + } + ) + + def dict_without_none(self) -> dict[str, Any]: + """ + Return model as dict, excluding None values + + Returns: + Dictionary with None values filtered out + """ + return {k: v for k, v in self.model_dump().items() if v is not None} diff --git a/src/clients/__init__.py b/src/clients/__init__.py new file mode 100644 index 0000000..ed7f801 --- /dev/null +++ b/src/clients/__init__.py @@ -0,0 +1,5 @@ +""" +API Clients package for Core-API + +Provides HTTP/WebSocket clients for external infrastructure services. +""" diff --git a/src/clients/ai_client.py b/src/clients/ai_client.py new file mode 100644 index 0000000..9ab377e --- /dev/null +++ b/src/clients/ai_client.py @@ -0,0 +1,197 @@ +""" +Core-AI HTTP Client + +Provides interface to Core-AI service for AI performance metrics. +""" +import httpx +from typing import Optional, Dict, List, Any +from src.logging_config import get_logger +from src.config import get_settings + +logger = get_logger(__name__) +settings = get_settings() + + +class CoreAIClient: + """ + HTTP client for Core-AI service + + Provides access to AI performance metrics, tool execution stats, + and memory system monitoring. + """ + + def __init__( + self, + base_url: Optional[str] = None, + timeout: int = 10 + ): + """ + Initialize Core-AI client + + Args: + base_url: Core-AI base URL (default from settings) + timeout: Request timeout in seconds + """ + self.base_url = (base_url or getattr(settings, 'core_ai_base_url', 'http://core-ai:8086')).rstrip("/") + self.timeout = timeout + self.client = httpx.AsyncClient(timeout=self.timeout) + + async def close(self): + """Close the HTTP client""" + await self.client.aclose() + + async def health_check(self) -> bool: + """ + Check if Core-AI service is accessible + + Returns: + True if accessible, False otherwise + """ + try: + response = await self.client.get(f"{self.base_url}/health") + return response.status_code == 200 + except Exception as e: + logger.error(f"Core-AI health check failed: {e}") + return False + + async def get_metrics(self) -> Dict[str, Any]: + """ + Get comprehensive AI performance metrics + + Returns: + Dict with agent performance, tool execution, memory stats + + Example: + { + "uptime_seconds": 3600, + "timestamp": "2025-12-03T20:00:00Z", + "agent": { + "total_requests": 100, + "avg_response_time_ms": 1250.5, + "p95_response_time_ms": 3200.0, + ... + }, + "tools": { + "total_calls": 250, + "success_rate": 0.98, + "top_tools": {...} + }, + "memory": { + "tier1_hit_rate": 0.85, + ... + }, + ... + } + """ + try: + response = await self.client.get(f"{self.base_url}/metrics") + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as e: + logger.error(f"Failed to get metrics: HTTP {e.response.status_code}") + raise + except Exception as e: + logger.error(f"Failed to get metrics: {e}") + raise + + async def get_recent_errors(self, limit: int = 20) -> List[Dict[str, Any]]: + """ + Get recent request errors + + Args: + limit: Maximum number of errors to return + + Returns: + List of error records with timestamps + + Example: + [ + { + "timestamp": "2025-12-03T19:45:12Z", + "agent_type": "pydantic", + "error": "Connection timeout", + "duration_ms": 5000 + }, + ... + ] + """ + try: + response = await self.client.get( + f"{self.base_url}/metrics/errors", + params={"limit": limit} + ) + response.raise_for_status() + data = response.json() + return data.get("errors", []) + except Exception as e: + logger.error(f"Failed to get recent errors: {e}") + raise + + async def get_tool_failures(self, limit: int = 20) -> List[Dict[str, Any]]: + """ + Get recent tool execution failures + + Args: + limit: Maximum number of failures to return + + Returns: + List of tool failure records + + Example: + [ + { + "timestamp": "2025-12-03T19:50:30Z", + "tool_name": "list_containers", + "error": "Connection refused", + "duration_ms": 150 + }, + ... + ] + """ + try: + response = await self.client.get( + f"{self.base_url}/metrics/tool-failures", + params={"limit": limit} + ) + response.raise_for_status() + data = response.json() + return data.get("failures", []) + except Exception as e: + logger.error(f"Failed to get tool failures: {e}") + raise + + async def reset_metrics(self) -> bool: + """ + Reset all metrics (admin operation) + + Returns: + True if successful + """ + try: + response = await self.client.post(f"{self.base_url}/metrics/reset") + response.raise_for_status() + logger.info("Successfully reset Core-AI metrics") + return True + except Exception as e: + logger.error(f"Failed to reset metrics: {e}") + raise + + async def __aenter__(self): + """Async context manager entry""" + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit""" + await self.close() + + +# Singleton instance +_ai_client: Optional[CoreAIClient] = None + + +def get_ai_client() -> CoreAIClient: + """Get singleton Core-AI client instance""" + global _ai_client + if _ai_client is None: + _ai_client = CoreAIClient() + return _ai_client diff --git a/src/clients/authentik_client.py b/src/clients/authentik_client.py new file mode 100644 index 0000000..c6cdb54 --- /dev/null +++ b/src/clients/authentik_client.py @@ -0,0 +1,302 @@ +""" +Authentik API Client + +Provides methods for interacting with Authentik Identity Provider API. +Used for managing applications, providers, and authentication flows. +""" +import httpx +from typing import Dict, List, Any, Optional +from functools import lru_cache +from src.logging_config import get_logger + +logger = get_logger(__name__) + + +class AuthentikClient: + """Client for Authentik API operations""" + + def __init__(self, base_url: str, api_token: str): + """ + Initialize Authentik client + + Args: + base_url: Authentik base URL (e.g., http://authentik-server:9000) + api_token: API token for authentication + """ + self.base_url = base_url.rstrip('/') + self.api_token = api_token + self.client = httpx.AsyncClient(timeout=30.0) + + async def _request(self, method: str, endpoint: str, **kwargs) -> Dict: + """Make authenticated API request using token auth""" + headers = kwargs.pop("headers", {}) + headers["Authorization"] = f"Bearer {self.api_token}" + + response = await self.client.request( + method, + f"{self.base_url}/api/v3/{endpoint.lstrip('/')}", + headers=headers, + **kwargs + ) + + if not response.is_success: + logger.error(f"API request failed: {response.status_code}") + logger.error(f"Response body: {response.text}") + + response.raise_for_status() + return response.json() + + async def health_check(self) -> bool: + """Check if Authentik is accessible""" + try: + response = await self.client.get(f"{self.base_url}/-/health/live/") + return response.status_code == 200 + except Exception as e: + logger.error(f"Authentik health check failed: {e}") + return False + + async def create_oauth2_provider( + self, + name: str, + client_id: str, + redirect_uris: List[str], + authorization_flow_slug: str = "default-provider-authorization-implicit-consent", + signing_key: Optional[str] = None + ) -> Dict: + """ + Create an OAuth2/OIDC provider + + Args: + name: Provider name + client_id: OAuth2 client ID + redirect_uris: List of allowed redirect URIs + authorization_flow_slug: Authorization flow slug (will be resolved to UUID) + signing_key: Signing key UUID (defaults to auto-selected) + + Returns: + Created provider data including client_secret + """ + # Get authorization flow UUID from slug + flows = await self.list_flows() + auth_flow_uuid = None + invalidation_flow_uuid = None + + for flow in flows: + if flow.get("slug") == authorization_flow_slug: + auth_flow_uuid = flow.get("pk") + if flow.get("slug") == "default-provider-invalidation-flow": + invalidation_flow_uuid = flow.get("pk") + + if not auth_flow_uuid: + raise ValueError(f"Authorization flow '{authorization_flow_slug}' not found") + if not invalidation_flow_uuid: + raise ValueError("Invalidation flow not found") + + # Get signing key if not provided + if not signing_key: + keys = await self._request("GET", "crypto/certificatekeypairs/") + # Find the self-signed cert + for key in keys.get("results", []): + if "authentik" in key.get("name", "").lower(): + signing_key = key.get("pk") + break + + if not signing_key and keys.get("results"): + signing_key = keys["results"][0]["pk"] + + # Format redirect URIs as objects with matching_mode + formatted_redirect_uris = [ + {"url": uri, "matching_mode": "strict"} + for uri in redirect_uris + ] + + provider_data = { + "name": name, + "authorization_flow": auth_flow_uuid, + "invalidation_flow": invalidation_flow_uuid, + "client_type": "confidential", + "client_id": client_id, + "redirect_uris": formatted_redirect_uris, + "signing_key": signing_key, + "sub_mode": "hashed_user_id", + "include_claims_in_id_token": True, + "issuer_mode": "per_provider", + "access_token_validity": "minutes=60", + "refresh_token_validity": "days=30", + "property_mappings": [] # Will use default mappings + } + + result = await self._request("POST", "providers/oauth2/", json=provider_data) + logger.info(f"Created OAuth2 provider: {name} (ID: {result.get('pk')})") + return result + + async def create_application( + self, + name: str, + slug: str, + provider_pk: int, + launch_url: Optional[str] = None, + icon_url: Optional[str] = None + ) -> Dict: + """ + Create an application + + Args: + name: Application display name + slug: Application slug (URL-safe identifier) + provider_pk: Primary key of the provider to use + launch_url: Optional launch URL + icon_url: Optional icon URL + + Returns: + Created application data + """ + app_data = { + "name": name, + "slug": slug, + "provider": provider_pk, + "meta_launch_url": launch_url or "", + "meta_icon": icon_url or "", + "policy_engine_mode": "any", + "open_in_new_tab": False + } + + result = await self._request("POST", "core/applications/", json=app_data) + logger.info(f"Created application: {name} (slug: {slug})") + return result + + async def get_provider_by_name(self, name: str) -> Optional[Dict]: + """Get OAuth2 provider by name""" + providers = await self._request("GET", "providers/oauth2/", params={"name": name}) + results = providers.get("results", []) + return results[0] if results else None + + async def get_application_by_slug(self, slug: str) -> Optional[Dict]: + """Get application by slug""" + apps = await self._request("GET", "core/applications/", params={"slug": slug}) + results = apps.get("results", []) + return results[0] if results else None + + async def list_flows(self) -> List[Dict]: + """List all authentication flows""" + result = await self._request("GET", "flows/instances/") + return result.get("results", []) + + async def create_proxy_provider( + self, + name: str, + external_host: str, + authorization_flow_slug: str = "default-provider-authorization-implicit-consent", + mode: str = "forward_single", + token_validity: int = 480 # 8 hours in minutes + ) -> Dict: + """ + Create a Proxy Provider for forward authentication + + Args: + name: Provider name + external_host: External URL (e.g., https://auth.schweitz.net) + authorization_flow_slug: Authorization flow slug + mode: Proxy mode (forward_single for forward auth) + token_validity: Token validity in minutes (default: 480 = 8 hours) + + Returns: + Created provider data + """ + # Get authorization flow UUID from slug + flows = await self.list_flows() + auth_flow_uuid = None + invalidation_flow_uuid = None + + for flow in flows: + if flow.get("slug") == authorization_flow_slug: + auth_flow_uuid = flow.get("pk") + if flow.get("slug") == "default-provider-invalidation-flow": + invalidation_flow_uuid = flow.get("pk") + + if not auth_flow_uuid: + raise ValueError(f"Authorization flow '{authorization_flow_slug}' not found") + if not invalidation_flow_uuid: + raise ValueError("Invalidation flow not found") + + provider_data = { + "name": name, + "authorization_flow": auth_flow_uuid, + "invalidation_flow": invalidation_flow_uuid, + "mode": mode, + "external_host": external_host, + "access_token_validity": f"minutes={token_validity}", + "refresh_token_validity": f"minutes={token_validity}", + "session_duration": f"seconds={token_validity * 60}", + "cookie_domain": "", # Will use the domain of each proxied site + "property_mappings": [] + } + + result = await self._request("POST", "providers/proxy/", json=provider_data) + logger.info(f"Created Proxy provider: {name} (ID: {result.get('pk')})") + return result + + async def get_provider_by_name_proxy(self, name: str) -> Optional[Dict]: + """Get Proxy provider by name""" + providers = await self._request("GET", "providers/proxy/", params={"name": name}) + results = providers.get("results", []) + return results[0] if results else None + + async def create_outpost( + self, + name: str, + type: str, + providers: List[int], + config: Optional[Dict] = None + ) -> Dict: + """ + Create an Authentik Outpost + + Args: + name: Outpost name + type: Outpost type (e.g., "proxy") + providers: List of provider PKs + config: Optional configuration overrides + + Returns: + Created outpost data + """ + outpost_data = { + "name": name, + "type": type, + "providers": providers, + "config": config or {}, + "service_connection": None # Will use local Docker + } + + result = await self._request("POST", "outposts/instances/", json=outpost_data) + logger.info(f"Created outpost: {name} (ID: {result.get('pk')})") + return result + + async def get_outpost_by_name(self, name: str) -> Optional[Dict]: + """Get outpost by name""" + outposts = await self._request("GET", "outposts/instances/", params={"name": name}) + results = outposts.get("results", []) + return results[0] if results else None + + async def close(self): + """Close HTTP client""" + await self.client.aclose() + + +@lru_cache() +def get_authentik_client() -> AuthentikClient: + """Get cached Authentik client instance""" + # Import credentials from gitignored module + try: + from src.credentials import AUTHENTIK_URL, AUTHENTIK_CORE_API_TOKEN + except ImportError: + # Fallback to environment variables if credentials.py doesn't exist + import os + AUTHENTIK_URL = os.getenv("AUTHENTIK_URL", "http://authentik-server:9000") + AUTHENTIK_CORE_API_TOKEN = os.getenv("AUTHENTIK_API_TOKEN", "") + + return AuthentikClient( + base_url=AUTHENTIK_URL, + api_token=AUTHENTIK_CORE_API_TOKEN + ) diff --git a/src/clients/kuma_client.py b/src/clients/kuma_client.py new file mode 100644 index 0000000..6069438 --- /dev/null +++ b/src/clients/kuma_client.py @@ -0,0 +1,561 @@ +""" +Uptime Kuma Socket.IO Client + +Provides interface to Uptime Kuma via Socket.IO for monitor management. +Also provides metrics API access for real-time status data. +""" +import socketio +import asyncio +import httpx +import re +from typing import Optional, Dict, List, Any +from src.logging_config import get_logger +from src.config import get_settings + +logger = get_logger(__name__) +settings = get_settings() + + +class KumaClient: + """ + Socket.IO client for Uptime Kuma + + Uses Socket.IO for real-time communication with Uptime Kuma. + """ + + def __init__( + self, + base_url: Optional[str] = None, + username: Optional[str] = None, + password: Optional[str] = None, + timeout: int = 30 + ): + """ + Initialize Kuma client + + Args: + base_url: Kuma base URL (default from settings) + username: Kuma username (default from settings) + password: Kuma password (default from settings) + timeout: Request timeout in seconds + """ + self.base_url = (base_url or settings.kuma_url).rstrip("/") + self.username = username or settings.kuma_username + self.password = password or settings.kuma_password + self.timeout = timeout + + self.sio = socketio.AsyncClient( + reconnection=True, + reconnection_attempts=3, + reconnection_delay=1, + ) + self._connected = False + self._authenticated = False + self._monitors_cache: Dict[int, Dict[str, Any]] = {} + + if not self.username or not self.password: + logger.warning("Uptime Kuma credentials not configured") + + async def _ensure_connected(self): + """Ensure we have an active connection and authentication""" + if not self._connected: + await self.connect() + if not self._authenticated: + await self.login() + + async def connect(self): + """Connect to Uptime Kuma Socket.IO server""" + if self._connected: + return + + try: + await self.sio.connect(self.base_url, transports=['websocket']) + self._connected = True + logger.info(f"Connected to Uptime Kuma at {self.base_url}") + except Exception as e: + logger.error(f"Failed to connect to Uptime Kuma: {e}") + raise + + async def disconnect(self): + """Disconnect from Uptime Kuma""" + if self._connected: + await self.sio.disconnect() + self._connected = False + self._authenticated = False + logger.info("Disconnected from Uptime Kuma") + + async def login(self): + """Authenticate with Uptime Kuma""" + if not self._connected: + await self.connect() + + try: + # Uptime Kuma login event + login_response = await self.sio.call( + 'login', + { + 'username': self.username, + 'password': self.password, + 'token': None + }, + timeout=self.timeout + ) + + if login_response and login_response.get('ok'): + self._authenticated = True + logger.info("Successfully authenticated with Uptime Kuma") + else: + error_msg = login_response.get('msg', 'Unknown error') if login_response else 'No response' + raise Exception(f"Login failed: {error_msg}") + + except Exception as e: + logger.error(f"Failed to authenticate with Uptime Kuma: {e}") + raise + + async def health_check(self) -> bool: + """ + Check if Uptime Kuma is accessible + + Returns: + True if accessible, False otherwise + """ + try: + await self._ensure_connected() + return self._authenticated + except Exception as e: + logger.error(f"Uptime Kuma health check failed: {e}") + return False + + async def get_monitors(self) -> List[Dict[str, Any]]: + """ + List all monitors with uptime data + + Returns: + List of monitor configurations with uptime_24h field + """ + await self._ensure_connected() + + try: + # Storage for monitor list and uptime data received via events + monitor_list_data = {} + uptime_list_data = {} + monitor_event_received = asyncio.Event() + uptime_event_received = asyncio.Event() + + # Register event handler for monitorList + @self.sio.event + async def monitorList(data): + nonlocal monitor_list_data + monitor_list_data = data + monitor_event_received.set() + + # Register event handler for uptimeList (24h uptime percentages) + @self.sio.event + async def uptimeList(monitor_id, uptime_data): + nonlocal uptime_list_data + # uptime_data is typically a dict with time periods: {"24": 99.5, "720": 98.2, ...} + uptime_list_data[str(monitor_id)] = uptime_data + # Don't set event here as we'll get multiple calls + + # Request monitor list - this triggers the server to send monitorList event + response = await self.sio.call('getMonitorList', timeout=self.timeout) + logger.info(f"getMonitorList call response: {response}") + + # Wait for the monitorList event (with timeout) + try: + await asyncio.wait_for(monitor_event_received.wait(), timeout=5.0) + logger.info(f"Received monitorList event with {len(monitor_list_data)} items") + + # Give time for uptimeList events to arrive + await asyncio.sleep(0.5) + logger.info(f"Received uptime data for {len(uptime_list_data)} monitors") + except asyncio.TimeoutError: + logger.warning("Timeout waiting for monitorList event") + + # Process the monitor list data + if monitor_list_data and isinstance(monitor_list_data, dict): + monitors = [] + for monitor_id, monitor_data in monitor_list_data.items(): + if isinstance(monitor_data, dict): + monitor_data['id'] = int(monitor_id) + + # Add uptime data if available + uptime_info = uptime_list_data.get(str(monitor_id), {}) + if isinstance(uptime_info, dict): + # Uptime Kuma provides 24h uptime as key "24" + monitor_data['uptime_24h'] = float(uptime_info.get('24', 0)) + else: + monitor_data['uptime_24h'] = 0.0 + + monitors.append(monitor_data) + self._monitors_cache[int(monitor_id)] = monitor_data + + logger.info(f"Found {len(monitors)} monitors total") + return monitors + + logger.warning(f"No valid monitor data received") + return [] + + except Exception as e: + logger.error(f"Failed to get monitors: {e}", exc_info=True) + raise + + async def get_monitor(self, monitor_id: int) -> Dict[str, Any]: + """ + Get details of a specific monitor + + Args: + monitor_id: Monitor identifier + + Returns: + Monitor configuration details + """ + await self._ensure_connected() + + try: + response = await self.sio.call('getMonitor', monitor_id, timeout=self.timeout) + + if response: + self._monitors_cache[monitor_id] = response + return response + + raise Exception(f"Monitor {monitor_id} not found") + + except Exception as e: + logger.error(f"Failed to get monitor {monitor_id}: {e}") + raise + + async def find_monitor_by_name(self, name: str) -> Optional[Dict[str, Any]]: + """ + Find a monitor by its name (case-insensitive) + + Args: + name: Monitor name to search for + + Returns: + Monitor object if found, None otherwise + """ + monitors = await self.get_monitors() + name_lower = name.lower() + + for monitor in monitors: + if monitor.get("name", "").lower() == name_lower: + return monitor + + return None + + async def find_monitors_by_tag(self, tag: str) -> List[Dict[str, Any]]: + """ + Find all monitors with a specific tag + + Args: + tag: Tag name to search for + + Returns: + List of monitors with the tag + """ + monitors = await self.get_monitors() + tagged_monitors = [] + + for monitor in monitors: + monitor_tags = monitor.get("tags", []) + if any(t.get("name", "").lower() == tag.lower() for t in monitor_tags): + tagged_monitors.append(monitor) + + return tagged_monitors + + async def pause_monitor(self, monitor_id: int) -> bool: + """ + Pause a monitor (disable monitoring) + + Args: + monitor_id: Monitor identifier + + Returns: + True if successful + """ + await self._ensure_connected() + + try: + # Uptime Kuma pause event + response = await self.sio.call('pauseMonitor', monitor_id, timeout=self.timeout) + + if response and response.get('ok'): + logger.info(f"Paused monitor {monitor_id}") + return True + + error_msg = response.get('msg', 'Unknown error') if response else 'No response' + raise Exception(f"Failed to pause monitor: {error_msg}") + + except Exception as e: + logger.error(f"Failed to pause monitor {monitor_id}: {e}") + raise + + async def resume_monitor(self, monitor_id: int) -> bool: + """ + Resume a monitor (enable monitoring) + + Args: + monitor_id: Monitor identifier + + Returns: + True if successful + """ + await self._ensure_connected() + + try: + # Uptime Kuma resume event + response = await self.sio.call('resumeMonitor', monitor_id, timeout=self.timeout) + + if response and response.get('ok'): + logger.info(f"Resumed monitor {monitor_id}") + return True + + error_msg = response.get('msg', 'Unknown error') if response else 'No response' + raise Exception(f"Failed to resume monitor: {error_msg}") + + except Exception as e: + logger.error(f"Failed to resume monitor {monitor_id}: {e}") + raise + + async def pause_monitor_by_name(self, name: str) -> bool: + """ + Pause a monitor by its name + + Args: + name: Monitor name + + Returns: + True if successful, False if monitor not found + """ + monitor = await self.find_monitor_by_name(name) + if not monitor: + logger.warning(f"Monitor '{name}' not found") + return False + + await self.pause_monitor(monitor["id"]) + return True + + async def resume_monitor_by_name(self, name: str) -> bool: + """ + Resume a monitor by its name + + Args: + name: Monitor name + + Returns: + True if successful, False if monitor not found + """ + monitor = await self.find_monitor_by_name(name) + if not monitor: + logger.warning(f"Monitor '{name}' not found") + return False + + await self.resume_monitor(monitor["id"]) + return True + + async def add_monitor(self, monitor_config: Dict[str, Any]) -> Dict[str, Any]: + """ + Create a new monitor + + Args: + monitor_config: Monitor configuration dict + + Returns: + Created monitor details including ID + """ + await self._ensure_connected() + + try: + # Uptime Kuma add monitor event + response = await self.sio.call('add', monitor_config, timeout=self.timeout) + + if response and response.get('ok'): + monitor_id = response.get('monitorID') + logger.info(f"Created monitor '{monitor_config.get('name')}' with ID {monitor_id}") + + # Get full monitor details + monitor = await self.get_monitor(monitor_id) + return monitor + + error_msg = response.get('msg', 'Unknown error') if response else 'No response' + raise Exception(f"Failed to create monitor: {error_msg}") + + except Exception as e: + logger.error(f"Failed to create monitor '{monitor_config.get('name')}': {e}") + raise + + async def update_monitor(self, monitor_id: int, monitor_config: Dict[str, Any]) -> Dict[str, Any]: + """ + Update an existing monitor + + Args: + monitor_id: Monitor identifier + monitor_config: Updated monitor configuration + + Returns: + Updated monitor details + """ + await self._ensure_connected() + + try: + # Ensure ID is in the config + monitor_config['id'] = monitor_id + + # Uptime Kuma edit monitor event + response = await self.sio.call('editMonitor', monitor_config, timeout=self.timeout) + + if response and response.get('ok'): + logger.info(f"Updated monitor {monitor_id}") + + # Get updated monitor details + monitor = await self.get_monitor(monitor_id) + return monitor + + error_msg = response.get('msg', 'Unknown error') if response else 'No response' + raise Exception(f"Failed to update monitor: {error_msg}") + + except Exception as e: + logger.error(f"Failed to update monitor {monitor_id}: {e}") + raise + + async def delete_monitor(self, monitor_id: int) -> bool: + """ + Delete a monitor + + Args: + monitor_id: Monitor identifier + + Returns: + True if successful + """ + await self._ensure_connected() + + try: + # Uptime Kuma delete monitor event + response = await self.sio.call('deleteMonitor', monitor_id, timeout=self.timeout) + + if response and response.get('ok'): + logger.info(f"Deleted monitor {monitor_id}") + + # Remove from cache + self._monitors_cache.pop(monitor_id, None) + return True + + error_msg = response.get('msg', 'Unknown error') if response else 'No response' + raise Exception(f"Failed to delete monitor: {error_msg}") + + except Exception as e: + logger.error(f"Failed to delete monitor {monitor_id}: {e}") + raise + + async def delete_monitor_by_name(self, name: str) -> bool: + """ + Delete a monitor by its name + + Args: + name: Monitor name + + Returns: + True if successful, False if monitor not found + """ + monitor = await self.find_monitor_by_name(name) + if not monitor: + logger.warning(f"Monitor '{name}' not found") + return False + + await self.delete_monitor(monitor["id"]) + return True + + async def get_metrics_status(self) -> Dict[str, Dict[str, Any]]: + """ + Get monitor status from Prometheus metrics endpoint + + This is simpler and more reliable than Socket.IO for getting current status. + Returns real-time UP/DOWN status but not historical uptime percentages. + + Returns: + Dict mapping monitor names to status info: + { + "Portainer": { + "status": 1, # 1=UP, 0=DOWN, 2=PENDING, 3=MAINTENANCE + "response_time": 5, # ms + "monitor_type": "http", + "url": "http://192.168.86.149:8001" + }, + ... + } + """ + try: + # Use API key authentication + api_key = settings.kuma_api_key + if not api_key: + logger.warning("Kuma API key not configured") + return {} + + # Fetch metrics with HTTP Basic Auth (empty username, API key as password) + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get( + f"{self.base_url}/metrics", + auth=("", api_key) + ) + response.raise_for_status() + metrics_text = response.text + + # Parse Prometheus format metrics + # Format: metric_name{label1="value1",label2="value2"} value + monitor_data = {} + + # Parse monitor_status lines + status_pattern = r'monitor_status\{monitor_name="([^"]+)",.*?\} (\d+)' + for match in re.finditer(status_pattern, metrics_text): + monitor_name = match.group(1) + status = int(match.group(2)) + + if monitor_name not in monitor_data: + monitor_data[monitor_name] = {} + monitor_data[monitor_name]['status'] = status + + # Parse monitor_response_time lines + response_pattern = r'monitor_response_time\{monitor_name="([^"]+)",monitor_type="([^"]+)",monitor_url="([^"]+)",.*?\} ([\d.]+)' + for match in re.finditer(response_pattern, metrics_text): + monitor_name = match.group(1) + monitor_type = match.group(2) + monitor_url = match.group(3) + response_time = float(match.group(4)) + + if monitor_name not in monitor_data: + monitor_data[monitor_name] = {} + monitor_data[monitor_name].update({ + 'response_time': response_time, + 'monitor_type': monitor_type, + 'url': monitor_url + }) + + logger.info(f"Fetched metrics for {len(monitor_data)} monitors") + return monitor_data + + except Exception as e: + logger.error(f"Failed to fetch metrics: {e}") + return {} + + async def __aenter__(self): + """Async context manager entry""" + await self._ensure_connected() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit""" + await self.disconnect() + + +# Singleton instance +_kuma_client: Optional[KumaClient] = None + + +def get_kuma_client() -> KumaClient: + """Get singleton Kuma client instance""" + global _kuma_client + if _kuma_client is None: + _kuma_client = KumaClient() + return _kuma_client diff --git a/src/clients/npm_client.py b/src/clients/npm_client.py new file mode 100644 index 0000000..14629f4 --- /dev/null +++ b/src/clients/npm_client.py @@ -0,0 +1,383 @@ +""" +Nginx Proxy Manager API Client + +Provides interface to NPM REST API for proxy host and SSL certificate management. +""" +import httpx +from typing import Optional, Dict, List, Any +from datetime import datetime, timedelta +from src.logging_config import get_logger +from src.config import get_settings + +logger = get_logger(__name__) +settings = get_settings() + + +class NPMClient: + """ + HTTP client for Nginx Proxy Manager API + + Uses JWT Bearer token authentication with automatic token refresh. + Tokens expire after ~24 hours. + """ + + def __init__( + self, + base_url: Optional[str] = None, + email: Optional[str] = None, + password: Optional[str] = None, + timeout: int = 30 + ): + """ + Initialize NPM client + + Args: + base_url: NPM base URL (default from settings) + email: NPM admin email (default from settings) + password: NPM admin password (default from settings) + timeout: Request timeout in seconds + """ + self.base_url = (base_url or settings.npm_url).rstrip("/") + self.email = email or settings.npm_email + self.password = password or settings.npm_password + self.timeout = timeout + + self._token: Optional[str] = None + self._token_expires: Optional[datetime] = None + + if not self.email or not self.password: + logger.warning("NPM credentials not configured") + + async def _ensure_token(self): + """Ensure we have a valid token, refresh if needed""" + if self._token and self._token_expires: + # If token expires in less than 1 hour, refresh it + if datetime.now() + timedelta(hours=1) < self._token_expires: + return + + # Get new token + await self._refresh_token() + + async def _refresh_token(self): + """Get a new authentication token""" + try: + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + f"{self.base_url}/api/tokens", + json={ + "identity": self.email, + "secret": self.password + } + ) + response.raise_for_status() + data = response.json() + + self._token = data.get("token") + # Assume 23-hour expiration to be safe + self._token_expires = datetime.now() + timedelta(hours=23) + + logger.info("NPM token refreshed successfully") + except Exception as e: + logger.error(f"Failed to refresh NPM token: {e}") + raise + + def _get_headers(self) -> Dict[str, str]: + """Get request headers with authentication""" + if not self._token: + raise RuntimeError("No NPM token available. Call _ensure_token() first.") + + return { + "Authorization": f"Bearer {self._token}", + "Content-Type": "application/json" + } + + async def health_check(self) -> bool: + """ + Check if NPM API is accessible + + Returns: + True if accessible, False otherwise + """ + try: + async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client: + response = await client.get(f"{self.base_url}/api") + # Accept any successful response (2xx) or redirect (3xx) as healthy + # A redirect indicates the service is up and responding + return 200 <= response.status_code < 400 + except Exception as e: + logger.error(f"NPM health check failed: {e}") + return False + + async def get_proxy_hosts(self) -> List[Dict[str, Any]]: + """ + List all proxy hosts + + Returns: + List of proxy host configurations + """ + await self._ensure_token() + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.base_url}/api/nginx/proxy-hosts", + headers=self._get_headers() + ) + response.raise_for_status() + return response.json() + + async def get_proxy_host(self, host_id: int) -> Dict[str, Any]: + """ + Get details of a specific proxy host + + Args: + host_id: Proxy host identifier + + Returns: + Proxy host configuration + """ + await self._ensure_token() + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.base_url}/api/nginx/proxy-hosts/{host_id}", + headers=self._get_headers() + ) + response.raise_for_status() + return response.json() + + async def create_proxy_host( + self, + domain_names: List[str], + forward_host: str, + forward_port: int, + forward_scheme: str = "http", + certificate_id: int = 0, + ssl_forced: bool = False, + block_exploits: bool = True, + caching_enabled: bool = True, + websocket_upgrade: bool = True, + http2_support: bool = True, + hsts_enabled: bool = True, + advanced_config: str = "" + ) -> Dict[str, Any]: + """ + Create a new proxy host + + Args: + domain_names: List of domain names for this proxy + forward_host: Target host to proxy to + forward_port: Target port to proxy to + forward_scheme: http or https + certificate_id: SSL certificate ID (0 for none) + ssl_forced: Force HTTPS redirect + block_exploits: Enable exploit blocking + caching_enabled: Enable response caching + websocket_upgrade: Allow WebSocket upgrades + http2_support: Enable HTTP/2 + hsts_enabled: Enable HSTS headers + advanced_config: Custom nginx configuration + + Returns: + Created proxy host details + """ + await self._ensure_token() + + payload = { + "domain_names": domain_names, + "forward_scheme": forward_scheme, + "forward_host": forward_host, + "forward_port": forward_port, + "certificate_id": certificate_id, + "ssl_forced": ssl_forced, + "block_exploits": block_exploits, + "caching_enabled": caching_enabled, + "allow_websocket_upgrade": websocket_upgrade, + "http2_support": http2_support, + "hsts_enabled": hsts_enabled, + "hsts_subdomains": False, + "advanced_config": advanced_config, + "access_list_id": 0, + "meta": {} + } + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + f"{self.base_url}/api/nginx/proxy-hosts", + headers=self._get_headers(), + json=payload + ) + response.raise_for_status() + return response.json() + + async def update_proxy_host( + self, + proxy_id: int, + config: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Update an existing proxy host configuration + + Args: + proxy_id: Proxy host ID to update + config: Full proxy host configuration (get from get_proxy_host, modify, then update) + + Returns: + Updated proxy host details + """ + await self._ensure_token() + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.put( + f"{self.base_url}/api/nginx/proxy-hosts/{proxy_id}", + headers=self._get_headers(), + json=config + ) + + if not response.is_success: + logger.error(f"Update failed: {response.status_code}") + logger.error(f"Response: {response.text}") + + response.raise_for_status() + return response.json() + + async def enable_authentik_forward_auth( + self, + proxy_id: int, + authentik_url: str = "http://authentik-server:9000" + ) -> Dict[str, Any]: + """ + Enable Authentik forward authentication on a proxy host + + Args: + proxy_id: Proxy host ID to update + authentik_url: Authentik server URL (default: http://authentik-server:9000) + + Returns: + Updated proxy host details + """ + # Get current config + proxy_host = await self.get_proxy_host(proxy_id) + + # Authentik forward auth configuration + auth_config = f"""# Authentik Forward Authentication +# Send authentication requests to Authentik +auth_request /outpost.goauthentik.io/auth/nginx; + +# Preserve authentication cookies +auth_request_set $auth_cookie $upstream_http_set_cookie; +add_header Set-Cookie $auth_cookie; + +# Get user information from Authentik +auth_request_set $authentik_username $upstream_http_x_authentik_username; +auth_request_set $authentik_groups $upstream_http_x_authentik_groups; +auth_request_set $authentik_email $upstream_http_x_authentik_email; +auth_request_set $authentik_name $upstream_http_x_authentik_name; +auth_request_set $authentik_uid $upstream_http_x_authentik_uid; + +# Pass user info to backend +proxy_set_header X-authentik-username $authentik_username; +proxy_set_header X-authentik-groups $authentik_groups; +proxy_set_header X-authentik-email $authentik_email; +proxy_set_header X-authentik-name $authentik_name; +proxy_set_header X-authentik-uid $authentik_uid; + +# On authentication failure, redirect to Authentik login +error_page 401 = @authentik_proxy_signin; + +location @authentik_proxy_signin {{ + internal; + add_header Set-Cookie $auth_cookie; + return 302 /outpost.goauthentik.io/start?rd=$scheme://$http_host$request_uri; +}} + +# Authentik authentication endpoint +location /outpost.goauthentik.io {{ + proxy_pass {authentik_url}/outpost.goauthentik.io; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header Host $host; +}} +""" + + # Update the advanced config + proxy_host["advanced_config"] = auth_config + + # Remove read-only fields that NPM doesn't accept in updates + readonly_fields = [ + "id", "created_on", "modified_on", "owner", "owner_user_id", + "certificate", "use_default_location", "ipv6", "meta", "nginx_online", + "nginx_err", "access_list", "certificate_id" + ] + + clean_config = {k: v for k, v in proxy_host.items() if k not in readonly_fields} + + # Ensure locations is an array (required field) + if "locations" not in clean_config or clean_config["locations"] is None: + clean_config["locations"] = [] + + # Update the proxy host + return await self.update_proxy_host(proxy_id, clean_config) + + async def get_certificates(self) -> List[Dict[str, Any]]: + """ + List all SSL certificates + + Returns: + List of certificate details + """ + await self._ensure_token() + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.base_url}/api/nginx/certificates", + headers=self._get_headers() + ) + response.raise_for_status() + return response.json() + + async def create_certificate( + self, + domain_names: List[str], + provider: str = "letsencrypt" + ) -> Dict[str, Any]: + """ + Request a new SSL certificate from Let's Encrypt + + Args: + domain_names: List of domains for the certificate + provider: Certificate provider (default: letsencrypt) + + Returns: + Certificate details + """ + await self._ensure_token() + + payload = { + "provider": provider, + "domain_names": domain_names, + "meta": { + "dns_challenge": False + } + } + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + f"{self.base_url}/api/nginx/certificates", + headers=self._get_headers(), + json=payload + ) + response.raise_for_status() + return response.json() + + +# Singleton instance +_npm_client: Optional[NPMClient] = None + + +def get_npm_client() -> NPMClient: + """Get singleton NPM client instance""" + global _npm_client + if _npm_client is None: + _npm_client = NPMClient() + return _npm_client diff --git a/src/clients/portainer_client.py b/src/clients/portainer_client.py new file mode 100644 index 0000000..ec9d60a --- /dev/null +++ b/src/clients/portainer_client.py @@ -0,0 +1,450 @@ +""" +Portainer API Client + +Provides interface to Portainer REST API for stack and container management. +Includes fallback to Docker socket for containers not managed by Portainer. +""" +import httpx +import json +from typing import Optional, Dict, List, Any +from src.logging_config import get_logger +from src.config import get_settings + +logger = get_logger(__name__) +settings = get_settings() + + +class PortainerClient: + """ + HTTP client for Portainer API + + Uses access token authentication (X-API-Key header) + for long-lived API access without session management. + """ + + def __init__( + self, + base_url: Optional[str] = None, + api_key: Optional[str] = None, + timeout: int = 30 + ): + """ + Initialize Portainer client + + Args: + base_url: Portainer base URL (default from settings) + api_key: Portainer API access token (default from settings) + timeout: Request timeout in seconds + """ + self.base_url = (base_url or settings.portainer_url).rstrip("/") + self.api_key = api_key or settings.portainer_api_key + self.timeout = timeout + + if not self.api_key: + logger.warning("Portainer API key not configured") + + def _get_headers(self) -> Dict[str, str]: + """Get request headers with authentication""" + return { + "X-API-Key": self.api_key, + "Content-Type": "application/json" + } + + async def health_check(self) -> bool: + """ + Check if Portainer API is accessible + + Returns: + True if accessible, False otherwise + """ + try: + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get(f"{self.base_url}/api/status") + return response.status_code == 200 + except Exception as e: + logger.error(f"Portainer health check failed: {e}") + return False + + async def get_endpoints(self) -> List[Dict[str, Any]]: + """ + List all Portainer endpoints (Docker environments) + + Returns: + List of endpoint configurations + """ + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.base_url}/api/endpoints", + headers=self._get_headers() + ) + response.raise_for_status() + return response.json() + + async def get_stacks(self, endpoint_id: Optional[int] = None) -> List[Dict[str, Any]]: + """ + List all stacks + + Args: + endpoint_id: Filter by specific endpoint (optional) + + Returns: + List of stack configurations + """ + params = {} + if endpoint_id: + params["endpointId"] = endpoint_id + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.base_url}/api/stacks", + headers=self._get_headers(), + params=params + ) + response.raise_for_status() + return response.json() + + async def get_stack(self, stack_id: int) -> Dict[str, Any]: + """ + Get details of a specific stack + + Args: + stack_id: Stack identifier + + Returns: + Stack configuration details + """ + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.base_url}/api/stacks/{stack_id}", + headers=self._get_headers() + ) + response.raise_for_status() + return response.json() + + async def create_stack( + self, + name: str, + stack_file_content: str, + endpoint_id: int + ) -> Dict[str, Any]: + """ + Create a new stack from compose file content + + Args: + name: Stack name + stack_file_content: Docker Compose YAML content + endpoint_id: Portainer endpoint to deploy to + + Returns: + Created stack details + """ + payload = { + "name": name, + "stackFileContent": stack_file_content + } + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + f"{self.base_url}/api/stacks/create/standalone/string", + headers=self._get_headers(), + params={"endpointId": endpoint_id}, + json=payload + ) + response.raise_for_status() + return response.json() + + async def update_stack( + self, + stack_id: int, + stack_file_content: str, + endpoint_id: int, + prune: bool = False, + pull_image: bool = False + ) -> Dict[str, Any]: + """ + Update an existing stack + + Args: + stack_id: Stack identifier + stack_file_content: New Docker Compose YAML content + endpoint_id: Portainer endpoint + prune: Remove services no longer defined + pull_image: Pull latest images before deployment + + Returns: + Updated stack details + """ + payload = { + "stackFileContent": stack_file_content, + "prune": prune, + "pullImage": pull_image + } + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.put( + f"{self.base_url}/api/stacks/{stack_id}", + headers=self._get_headers(), + params={"endpointId": endpoint_id}, + json=payload + ) + response.raise_for_status() + return response.json() + + async def delete_stack(self, stack_id: int, endpoint_id: int) -> bool: + """ + Delete a stack + + Args: + stack_id: Stack identifier + endpoint_id: Portainer endpoint + + Returns: + True if successful + """ + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.delete( + f"{self.base_url}/api/stacks/{stack_id}", + headers=self._get_headers(), + params={"endpointId": endpoint_id} + ) + response.raise_for_status() + return True + + async def get_containers(self, endpoint_id: int, all_containers: bool = True) -> List[Dict[str, Any]]: + """ + List containers on a specific endpoint + + Args: + endpoint_id: Portainer endpoint identifier + all_containers: Include stopped containers (default: True) + + Returns: + List of container details + """ + params = {"all": 1 if all_containers else 0} + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.base_url}/api/endpoints/{endpoint_id}/docker/containers/json", + headers=self._get_headers(), + params=params + ) + response.raise_for_status() + return response.json() + + async def get_container(self, endpoint_id: int, container_id: str) -> Dict[str, Any]: + """ + Get detailed information about a specific container + + Args: + endpoint_id: Portainer endpoint identifier + container_id: Container ID or name + + Returns: + Container details including network and port information + """ + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.base_url}/api/endpoints/{endpoint_id}/docker/containers/{container_id}/json", + headers=self._get_headers() + ) + response.raise_for_status() + return response.json() + + async def stop_container(self, endpoint_id: int, container_id: str) -> bool: + """ + Stop a container + + Args: + endpoint_id: Portainer endpoint identifier + container_id: Container ID or name + + Returns: + True if successful + """ + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + f"{self.base_url}/api/endpoints/{endpoint_id}/docker/containers/{container_id}/stop", + headers=self._get_headers() + ) + response.raise_for_status() + logger.info(f"Stopped container {container_id}") + return True + + async def start_container(self, endpoint_id: int, container_id: str) -> bool: + """ + Start a container + + Args: + endpoint_id: Portainer endpoint identifier + container_id: Container ID or name + + Returns: + True if successful + """ + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + f"{self.base_url}/api/endpoints/{endpoint_id}/docker/containers/{container_id}/start", + headers=self._get_headers() + ) + response.raise_for_status() + logger.info(f"Started container {container_id}") + return True + + # ======================================================================== + # Docker Socket Fallback (for containers not managed by Portainer) + # ======================================================================== + + async def _list_containers_via_socket(self, all_containers: bool = True) -> List[Dict[str, Any]]: + """ + Fallback: List containers directly via Docker socket + + Used when Portainer API doesn't return complete data (e.g., containers + started outside Portainer, AMP game servers, etc.) + + Args: + all_containers: Include stopped containers + + Returns: + List of container details in Docker API format + """ + try: + # Docker socket is mounted at /var/run/docker.sock + # Use httpx with unix socket transport + transport = httpx.AsyncHTTPTransport(uds="/var/run/docker.sock") + async with httpx.AsyncClient(transport=transport, timeout=10) as client: + params = {"all": 1 if all_containers else 0} + response = await client.get( + "http://localhost/v1.41/containers/json", + params=params + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.warning(f"Docker socket fallback failed: {e}") + return [] + + async def _inspect_container_via_socket(self, container_id_or_name: str) -> Optional[Dict[str, Any]]: + """ + Fallback: Inspect container directly via Docker socket + + Args: + container_id_or_name: Container ID or name + + Returns: + Container details or None + """ + try: + transport = httpx.AsyncHTTPTransport(uds="/var/run/docker.sock") + async with httpx.AsyncClient(transport=transport, timeout=10) as client: + response = await client.get( + f"http://localhost/v1.41/containers/{container_id_or_name}/json" + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.warning(f"Docker socket inspect fallback failed for '{container_id_or_name}': {e}") + return None + + # ======================================================================== + # Helper methods for agent tools (auto-detect endpoint + fallback) + # ======================================================================== + + async def list_containers(self, all_containers: bool = True) -> List[Dict[str, Any]]: + """ + List containers using auto-detected endpoint with Docker socket fallback + + This is a convenience wrapper that automatically uses the first/default endpoint. + If Portainer doesn't have complete data, falls back to Docker socket. + + Args: + all_containers: Include stopped containers (default: True) + + Returns: + List of container details + """ + try: + # Try Portainer first + endpoints = await self.get_endpoints() + if endpoints: + endpoint_id = endpoints[0]["Id"] + containers = await self.get_containers(endpoint_id, all_containers) + if containers: + return containers + + # Fallback to Docker socket + logger.info("Portainer returned no containers, trying Docker socket fallback...") + return await self._list_containers_via_socket(all_containers) + + except Exception as e: + logger.error(f"Error listing containers: {e}") + # Try fallback even on exception + try: + return await self._list_containers_via_socket(all_containers) + except Exception as fallback_error: + logger.error(f"Fallback also failed: {fallback_error}") + return [] + + async def inspect_container(self, container_name: str) -> Optional[Dict[str, Any]]: + """ + Inspect a container by name using auto-detected endpoint with Docker socket fallback + + This is a convenience wrapper that automatically uses the first/default endpoint. + If Portainer doesn't find the container, falls back to Docker socket. + + Args: + container_name: Container name (e.g., "jellyfin", "ollama") + + Returns: + Container details or None if not found + """ + try: + # Try Portainer first + endpoints = await self.get_endpoints() + if endpoints: + endpoint_id = endpoints[0]["Id"] + + # First list all containers to find the one matching the name + all_containers = await self.get_containers(endpoint_id, all_containers=True) + + matching_container = None + for container in all_containers: + # Container names come as array like ['/jellyfin'] + names = container.get('Names', []) + for name in names: + clean_name = name.lstrip('/') + if clean_name == container_name or clean_name.lower() == container_name.lower(): + matching_container = container + break + if matching_container: + break + + if matching_container: + # Get detailed info using container ID + container_id = matching_container['Id'] + return await self.get_container(endpoint_id, container_id) + + # Not found in Portainer, try Docker socket fallback + logger.info(f"Container '{container_name}' not found in Portainer, trying Docker socket fallback...") + return await self._inspect_container_via_socket(container_name) + + except Exception as e: + logger.error(f"Error inspecting container '{container_name}': {e}") + # Try fallback even on exception + try: + return await self._inspect_container_via_socket(container_name) + except Exception as fallback_error: + logger.error(f"Fallback also failed: {fallback_error}") + return None + + +# Singleton instance +_portainer_client: Optional[PortainerClient] = None + + +def get_portainer_client() -> PortainerClient: + """Get singleton Portainer client instance""" + global _portainer_client + if _portainer_client is None: + _portainer_client = PortainerClient() + return _portainer_client diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..88f6b33 --- /dev/null +++ b/src/config.py @@ -0,0 +1,157 @@ +""" +Global configuration for Core Code API +""" +from pydantic_settings import BaseSettings +from functools import lru_cache + +# Import infrastructure credentials from gitignored module +try: + from src.credentials import ( + PORTAINER_URL, PORTAINER_API_KEY, + NPM_URL, NPM_EMAIL, NPM_PASSWORD, + KUMA_URL, KUMA_USERNAME, KUMA_PASSWORD, KUMA_API_KEY, + BRAVE_SEARCH_API_KEY, + GOOGLE_SEARCH_API_KEY, GOOGLE_SEARCH_ENGINE_ID + ) +except ImportError: + # Fallback to empty strings if credentials.py doesn't exist + # (e.g., fresh clone before credentials setup) + PORTAINER_URL = "http://localhost:8001" + PORTAINER_API_KEY = "" + NPM_URL = "http://localhost:81" + NPM_EMAIL = "" + NPM_PASSWORD = "" + KUMA_URL = "http://localhost:3001" + KUMA_USERNAME = "" + KUMA_PASSWORD = "" + KUMA_API_KEY = "" + BRAVE_SEARCH_API_KEY = "" + GOOGLE_SEARCH_API_KEY = "" + GOOGLE_SEARCH_ENGINE_ID = "" + + +class Settings(BaseSettings): + """Global application settings""" + + # Application + app_name: str = "Core Code API" + app_version: str = "1.0.0" + debug: bool = False + + # Server + host: str = "0.0.0.0" + port: int = 8083 + + # CORS + cors_origins: list[str] = ["*"] + cors_credentials: bool = True + cors_methods: list[str] = ["*"] + cors_headers: list[str] = ["*"] + + # Logging + log_level: str = "DEBUG" + + # Ollama Configuration (for AI orchestration) + ollama_base_url: str = "http://ollama:11434" + ollama_timeout: int = 300 # 5 minutes + + # Model Configuration + default_model: str = "mistral-tools:7b" + agent_model: str = "gemma2:9b-instruct-q5_K_M" # Must support tool calling with ADK (~4GB VRAM) + lightweight_models: str = "gemma3-tools:1b,phi3:mini" + heavy_models: str = "mistral:7b,gemma2:9b,gemma3:12b,mixtral:8x7b" + code_models: str = "codestral:latest,codegemma:latest" + # Previous config (gemma3:12b used ~10GB VRAM) + # default_model: str = "gemma3:12b" + # agent_model: str = "gemma3:12b" + + # System Prompt Variant (for A/B testing) + # Options: v1_verbose, v2_concise, v3_imperative, v4_minimal, v4_gemini_suggestion, v5_adk_optimized, v7_adk_best_practice, v8_holistic + system_prompt_variant: str = "v8_holistic" + + # Agent Configuration + agent_fallback_enabled: bool = True + + # Model Aliases (OpenAI → Local) + alias_gpt35: str = "gemma:7b" + alias_gpt4: str = "mistral:7b" + alias_gpt4_turbo: str = "mixtral:8x7b" + alias_gpt4_code: str = "codestral:latest" + + # Memory Configuration + memory_tier1_max_turns: int = 10 + memory_consolidation_threshold: int = 10 + + # Qdrant Configuration + qdrant_host: str = "qdrant" + qdrant_port: int = 6333 + qdrant_collection_conversations: str = "core_api_conversations" + qdrant_collection_documents: str = "core_api_documents" + qdrant_collection_user_facts: str = "core_api_user_facts" + + # Embeddings (using Ollama - no local models needed) + embedding_model: str = "nomic-embed-text" # Ollama embedding model + embedding_dimension: int = 768 # nomic-embed-text dimension + embedding_batch_size: int = 32 + + # Search Configuration + search_provider: str = "google" # Options: google, brave, searxng, duckduckgo + searxng_url: str = "http://searxng:8080" # For future self-hosted SearxNG + + # Search API Keys (from credentials.py) + brave_search_api_key: str = BRAVE_SEARCH_API_KEY # https://brave.com/search/api/ + google_search_api_key: str = GOOGLE_SEARCH_API_KEY # https://console.cloud.google.com/ + google_search_engine_id: str = GOOGLE_SEARCH_ENGINE_ID # Custom Search Engine ID + + # Infrastructure Management (from credentials.py) + portainer_url: str = PORTAINER_URL + portainer_api_key: str = PORTAINER_API_KEY + + npm_url: str = NPM_URL + npm_email: str = NPM_EMAIL + npm_password: str = NPM_PASSWORD + + kuma_url: str = KUMA_URL + kuma_username: str = KUMA_USERNAME + kuma_password: str = KUMA_PASSWORD + kuma_api_key: str = KUMA_API_KEY + + # Core-AI Service (AI performance metrics) + core_ai_base_url: str = "http://core-ai:8086" + + # OIDC Authentication (Authentik) + oidc_enabled: bool = False # Set to True to require authentication + oidc_issuer: str = "https://auth.schweitz.net/application/o/core-api/" + oidc_audience: str = "core-api" + + @property + def model_aliases(self) -> dict: + """Computed property for model aliases""" + return { + "gpt-3.5-turbo": self.alias_gpt35, + "gpt-4": self.alias_gpt4, + "gpt-4-turbo": self.alias_gpt4_turbo, + "gpt-4-code": self.alias_gpt4_code, + } + + def get_lightweight_models(self) -> list[str]: + """Parse comma-separated lightweight models""" + return [m.strip().strip('"').strip("'") for m in self.lightweight_models.split(",") if m.strip()] + + def get_heavy_models(self) -> list[str]: + """Parse comma-separated heavy models""" + return [m.strip().strip('"').strip("'") for m in self.heavy_models.split(",") if m.strip()] + + def get_code_models(self) -> list[str]: + """Parse comma-separated code models""" + return [m.strip().strip('"').strip("'") for m in self.code_models.split(",") if m.strip()] + + class Config: + env_file = ".env" + case_sensitive = False + + +@lru_cache() +def get_settings() -> Settings: + """Cached settings instance""" + return Settings() diff --git a/src/controllers/__init__.py b/src/controllers/__init__.py new file mode 100644 index 0000000..c7593b0 --- /dev/null +++ b/src/controllers/__init__.py @@ -0,0 +1,5 @@ +""" +Controllers package for Core-API + +Provides controller-based routing architecture for better code organization. +""" diff --git a/src/controllers/ai_controller.py b/src/controllers/ai_controller.py new file mode 100644 index 0000000..2a52b9f --- /dev/null +++ b/src/controllers/ai_controller.py @@ -0,0 +1,219 @@ +""" +AI Metrics Proxy Controller + +Provides proxy endpoints to Core-AI service metrics. +Allows external access to AI performance stats via core-api. +""" +from fastapi import APIRouter, HTTPException +from typing import Dict, List, Any +from src.clients.ai_client import get_ai_client +from src.logging_config import get_logger + +logger = get_logger(__name__) + +# Create router +router = APIRouter( + prefix="/ai", + tags=["AI Metrics"] +) + + +@router.get( + "/health", + summary="Check Core-AI service health", + description="Verify that the Core-AI service is accessible and responding" +) +async def ai_health_check(): + """ + Check if Core-AI service is healthy + + Returns: + Health status and availability + """ + try: + ai_client = get_ai_client() + is_healthy = await ai_client.health_check() + + return { + "service": "core-ai", + "status": "healthy" if is_healthy else "unhealthy", + "accessible": is_healthy + } + + except Exception as e: + logger.error(f"AI health check failed: {e}") + return { + "service": "core-ai", + "status": "error", + "accessible": False, + "error": str(e) + } + + +@router.get( + "/metrics", + response_model=Dict[str, Any], + summary="Get comprehensive AI performance metrics", + description="Returns detailed metrics including agent performance, tool execution stats, memory system metrics, and user activity" +) +async def get_ai_metrics(): + """ + Proxy endpoint for Core-AI metrics + + Returns comprehensive AI performance data: + - Agent request statistics (total, by type, response times) + - Response time percentiles (p50, p95, p99) + - Tool execution metrics (calls, success rates, durations) + - Memory system statistics (cache hits, consolidations) + - User activity tracking + - Concurrency metrics + + Returns: + Dict with all collected metrics + + Raises: + HTTPException: If Core-AI is unreachable or returns error + """ + try: + ai_client = get_ai_client() + metrics = await ai_client.get_metrics() + return metrics + + except Exception as e: + logger.error(f"Failed to fetch AI metrics: {e}") + raise HTTPException( + status_code=503, + detail=f"Core-AI service unavailable: {str(e)}" + ) + + +@router.get( + "/metrics/errors", + response_model=Dict[str, Any], + summary="Get recent request errors", + description="Returns recent AI agent request errors with timestamps and details" +) +async def get_ai_errors(limit: int = 20): + """ + Get recent AI request errors + + Args: + limit: Maximum number of errors to return (default: 20) + + Returns: + Dict with error list and total count + + Example response: + { + "errors": [ + { + "timestamp": "2025-12-03T19:45:12Z", + "agent_type": "pydantic", + "error": "Connection timeout", + "duration_ms": 5000 + } + ], + "total": 1 + } + """ + try: + ai_client = get_ai_client() + errors = await ai_client.get_recent_errors(limit=limit) + + return { + "errors": errors, + "total": len(errors) + } + + except Exception as e: + logger.error(f"Failed to fetch AI errors: {e}") + raise HTTPException( + status_code=503, + detail=f"Core-AI service unavailable: {str(e)}" + ) + + +@router.get( + "/metrics/tool-failures", + response_model=Dict[str, Any], + summary="Get recent tool execution failures", + description="Returns recent tool execution failures with error details" +) +async def get_ai_tool_failures(limit: int = 20): + """ + Get recent tool execution failures + + Args: + limit: Maximum number of failures to return (default: 20) + + Returns: + Dict with failure list and total count + + Example response: + { + "failures": [ + { + "timestamp": "2025-12-03T19:50:30Z", + "tool_name": "list_containers", + "error": "Connection refused", + "duration_ms": 150 + } + ], + "total": 1 + } + """ + try: + ai_client = get_ai_client() + failures = await ai_client.get_tool_failures(limit=limit) + + return { + "failures": failures, + "total": len(failures) + } + + except Exception as e: + logger.error(f"Failed to fetch tool failures: {e}") + raise HTTPException( + status_code=503, + detail=f"Core-AI service unavailable: {str(e)}" + ) + + +@router.post( + "/metrics/reset", + summary="Reset all AI metrics (admin)", + description="Clear all collected metrics. This is an administrative operation that resets all counters and history." +) +async def reset_ai_metrics(): + """ + Reset all AI metrics (admin operation) + + Clears all collected metrics including: + - Request history + - Tool execution stats + - Memory system metrics + - Error logs + + Returns: + Success confirmation + + Note: + This is an administrative operation that should be used carefully. + All historical data will be lost. + """ + try: + ai_client = get_ai_client() + await ai_client.reset_metrics() + + logger.info("AI metrics reset successfully") + return { + "success": True, + "message": "AI metrics reset successfully" + } + + except Exception as e: + logger.error(f"Failed to reset AI metrics: {e}") + raise HTTPException( + status_code=503, + detail=f"Core-AI service unavailable: {str(e)}" + ) diff --git a/src/controllers/base.py b/src/controllers/base.py new file mode 100644 index 0000000..c2e8ce2 --- /dev/null +++ b/src/controllers/base.py @@ -0,0 +1,50 @@ +""" +Base controller class for Core-API + +Provides common functionality for all controllers. +""" +from fastapi import APIRouter +from abc import ABC, abstractmethod + + +class BaseController(ABC): + """ + Base controller class with common functionality + + All controllers should inherit from this class and implement + the create_router() method to define their endpoints. + """ + + def __init__(self, prefix: str, tags: list[str]): + """ + Initialize base controller + + Args: + prefix: URL prefix for this controller's routes + tags: OpenAPI tags for documentation grouping + """ + self.prefix = prefix + self.tags = tags + self._router = None + + @abstractmethod + def create_router(self) -> APIRouter: + """ + Create and configure the FastAPI router for this controller + + Returns: + Configured APIRouter instance with all endpoints + """ + pass + + @property + def router(self) -> APIRouter: + """ + Get the router instance, creating it if needed + + Returns: + APIRouter instance + """ + if self._router is None: + self._router = self.create_router() + return self._router diff --git a/src/controllers/health_controller.py b/src/controllers/health_controller.py new file mode 100644 index 0000000..ff21ad0 --- /dev/null +++ b/src/controllers/health_controller.py @@ -0,0 +1,248 @@ +""" +Health Controller + +Provides service health and information endpoints +""" +from fastapi import APIRouter, Response +from fastapi.responses import JSONResponse + +from src.controllers.base import BaseController +from src.config import get_settings +from src.logging_config import get_logger +from src.models.ollama_client import get_ollama_client + +# Note: Agent functionality moved to separate core-ai service (Dec 2025) +# This service (core-api) only provides infrastructure management and tools +AGENT_AVAILABLE = False + + +logger = get_logger(__name__) + + +class HealthController(BaseController): + """ + Controller for service health and information + + Provides endpoints for: + - Service information and status + - Health checks + """ + + def __init__(self): + super().__init__(prefix="", tags=["Health"]) + + def create_router(self) -> APIRouter: + """Create and configure the router""" + router = APIRouter(tags=self.tags) + settings = get_settings() + + @router.get( + "/", + summary="Service information", + response_class=JSONResponse + ) + async def root(): + """ + Get service information and health status + + Returns basic information about the API service and available endpoints. + """ + logger.debug("Root endpoint accessed") + return { + "service": settings.app_name, + "version": settings.app_version, + "status": "healthy", + "documentation": { + "swagger_ui": "/docs", + "redoc": "/redoc", + "openapi_spec": "/openapi.json" + }, + "endpoints": { + "chat_completions": "/v1/chat/completions", + "models": "/v1/models", + "conversations": "/v1/conversations", + "web_scraper": "/web-scraper/scrape", + "infrastructure": "/infrastructure", + "health": "/health", + "health_full": "/health/full" + } + } + + @router.get( + "/health", + summary="Health check", + response_class=JSONResponse + ) + async def health_check(): + """ + Simple health check endpoint for container orchestration + + Returns a 200 OK status when the service is running properly. + Used by Docker, Kubernetes, and load balancers. + """ + ollama_client = get_ollama_client() + ollama_healthy = await ollama_client.health_check() + + return { + "status": "healthy", + "ollama_connected": ollama_healthy + } + + @router.get( + "/health/full", + summary="Fast health check for Docker", + ) + async def full_health_check(response: Response): + """ + Fast health check for container orchestration (Docker/K8s). + + Checks component availability WITHOUT running expensive operations. + Returns 200 OK if all components are available, otherwise 503. + + For detailed diagnostics, use /health/diagnostics instead. + """ + import time + start_time = time.time() + + # Check 1: Ollama connection + verify agent model is available + ollama_client = get_ollama_client() + ollama_healthy = False + ollama_error = None + model_available = False + + try: + # Ping Ollama + ollama_healthy = await ollama_client.health_check() + + # Verify the agent model is pulled and check what's currently loaded + models_info = {} + if ollama_healthy: + try: + models_response = await ollama_client.list_models() + available_models = [m.get('name', '') for m in models_response.get('models', [])] + model_available = settings.agent_model in available_models + + # Get info about currently loaded models (those with size in memory) + loaded_models = [ + m.get('name', '') for m in models_response.get('models', []) + if m.get('size', 0) > 0 + ] + + models_info = { + "configured": settings.agent_model, + "available": model_available, + "total_in_ollama": len(available_models), + "currently_loaded": loaded_models if loaded_models else ["none"] + } + + if not model_available: + ollama_error = f"Model '{settings.agent_model}' not found in Ollama. Available: {', '.join(available_models[:3])}" + ollama_healthy = False + except Exception as e: + ollama_error = f"Could not list Ollama models: {str(e)}" + ollama_healthy = False + + except Exception as e: + ollama_error = str(e) + logger.warning(f"Ollama health check failed: {ollama_error}") + + # Note: Agent functionality moved to separate core-ai service + # This service only needs Ollama for embeddings (infrastructure tools) + # Agent health is checked separately in core-ai service + + # Determine overall status (only Ollama required for core-api) + is_healthy = ollama_healthy + + elapsed_ms = int((time.time() - start_time) * 1000) + status_code = 200 if is_healthy else 503 + response.status_code = status_code + + return { + "status": "healthy" if is_healthy else "unhealthy", + "status_code": status_code, + "response_time_ms": elapsed_ms, + "components": { + "ollama": { + "status": "✅ healthy" if ollama_healthy else "❌ unhealthy", + "models": models_info if models_info else { + "configured": settings.agent_model, + "available": False + }, + "error": ollama_error + }, + "note": "AI agent functionality available in separate core-ai service (port 8086)" + } + } + + @router.get( + "/health/diagnostics", + summary="Detailed system diagnostics", + ) + async def diagnostics(deep_test: bool = False): + """ + Comprehensive system diagnostics with detailed component information. + + Query Parameters: + - deep_test: Set to true to actually test agent generation (slow, ~5-10s) + + Returns detailed information about all system components. + """ + import time + + start_time = time.time() + diagnostics = { + "timestamp": time.time(), + "service": { + "name": settings.app_name, + "version": settings.app_version, + "purpose": "Infrastructure management and tools API" + }, + "components": {} + } + + # 1. Ollama Connection + ollama_client = get_ollama_client() + try: + ollama_healthy = await ollama_client.health_check() + diagnostics["components"]["ollama"] = { + "status": "✅ connected", + "url": settings.ollama_base_url, + "timeout": settings.ollama_timeout, + "default_model": settings.default_model + } + except Exception as e: + diagnostics["components"]["ollama"] = { + "status": "❌ error", + "error": str(e) + } + + # 2. Agent Stack - Moved to separate core-ai service + diagnostics["components"]["agent"] = { + "status": "N/A", + "note": "AI agent functionality moved to separate core-ai service (port 8086)", + "check_url": "http://core-ai:8086/health" + } + + # 3. Memory System (Qdrant) - Moved to core-ai service + diagnostics["components"]["qdrant"] = { + "status": "N/A", + "note": "Memory system managed by core-ai service (port 8086)" + } + + # 4. Configuration + diagnostics["configuration"] = { + "agent_fallback_enabled": settings.agent_fallback_enabled, + "memory_tier1_max_turns": settings.memory_tier1_max_turns, + "cors_origins": settings.cors_origins[:2] if len(settings.cors_origins) > 2 else settings.cors_origins + } + + elapsed_ms = int((time.time() - start_time) * 1000) + diagnostics["response_time_ms"] = elapsed_ms + + return diagnostics + + return router + + +# Create controller instance +health_controller = HealthController() diff --git a/src/controllers/infrastructure_controller.py b/src/controllers/infrastructure_controller.py new file mode 100644 index 0000000..a5a180d --- /dev/null +++ b/src/controllers/infrastructure_controller.py @@ -0,0 +1,1933 @@ +""" +Infrastructure Management Controller + +Provides API endpoints for automated infrastructure management, +including service deployment, configuration, and monitoring setup. +""" +from fastapi import APIRouter, HTTPException, Depends +from typing import List, Dict, Any, Optional, Union +from pydantic import BaseModel, field_validator + +from src.controllers.base import BaseController +from src.clients.portainer_client import get_portainer_client +from src.clients.npm_client import get_npm_client +from src.clients.kuma_client import get_kuma_client +from src.logging_config import get_logger +from src import service_groups +from src.auth.oidc import get_admin_user, get_forward_auth_admin + +logger = get_logger(__name__) + + +# Response models +class ServiceInfo(BaseModel): + """Information about a deployed service""" + name: str + stack_id: Optional[int] + status: Union[str, int] + endpoint_id: Optional[int] + ports: List[int] = [] + domains: List[str] = [] + running: bool = False # True if at least one container is running + containers_running: int = 0 # Number of running containers + containers_total: int = 0 # Total number of containers + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert status to string representation""" + if isinstance(v, int): + # Portainer status: 1=active, 2=inactive + return "active" if v == 1 else "inactive" + return v + + +class PortInfo(BaseModel): + """Information about an allocated port""" + port: int + service: str + container_name: Optional[str] = None + protocol: str = "tcp" + internal_hostname: Optional[str] = None + internal_ip: Optional[str] = None + external_domains: List[str] = [] + host_port: Optional[int] = None # Port exposed on host, if different from internal + description: str = "" + + +class DomainInfo(BaseModel): + """Information about a configured domain""" + domain: str + service: str + proxy_host_id: Optional[int] + ssl_enabled: bool = False + certificate_id: Optional[int] + + +class InfrastructureHealth(BaseModel): + """Overall infrastructure health status""" + portainer_connected: bool + npm_connected: bool + total_stacks: int + total_proxy_hosts: int + + +# Request models for write operations +class DeployServiceRequest(BaseModel): + """Request to deploy a new service""" + name: str + compose_content: str + endpoint_id: int = 3 # Default to local endpoint + + +class UpdateServiceRequest(BaseModel): + """Request to update an existing service""" + compose_content: str + prune: bool = False + pull_image: bool = True + + +class CreateProxyRequest(BaseModel): + """Request to create a new proxy host""" + domain_names: List[str] + forward_host: str + forward_port: int + forward_scheme: str = "http" + ssl_enabled: bool = False + request_ssl_certificate: bool = False + block_exploits: bool = True + websocket_upgrade: bool = True + http2_support: bool = True + + +class OperationResult(BaseModel): + """Result of an infrastructure operation""" + success: bool + message: str + details: Optional[Dict[str, Any]] = None + + +class InfrastructureController(BaseController): + """ + Controller for infrastructure management operations + + Provides endpoints for: + - Service discovery and listing + - Port allocation management + - Domain/proxy configuration + - Automated service deployment + """ + + def __init__(self): + super().__init__(prefix="/infrastructure", tags=["Infrastructure"]) + + def create_router(self) -> APIRouter: + """Create and configure the router""" + router = APIRouter(prefix=self.prefix, tags=self.tags) + + @router.get( + "/health", + response_model=InfrastructureHealth, + summary="Infrastructure health check" + ) + async def get_infrastructure_health(): + """ + Check health of all infrastructure services + + Returns status of Portainer, NPM, and summary statistics. + """ + portainer = get_portainer_client() + npm = get_npm_client() + + portainer_healthy = await portainer.health_check() + npm_healthy = await npm.health_check() + + total_stacks = 0 + total_proxy_hosts = 0 + + if portainer_healthy: + try: + stacks = await portainer.get_stacks() + total_stacks = len(stacks) + except Exception as e: + logger.error(f"Failed to get stacks count: {e}") + + if npm_healthy: + try: + proxy_hosts = await npm.get_proxy_hosts() + total_proxy_hosts = len(proxy_hosts) + except Exception as e: + logger.error(f"Failed to get proxy hosts count: {e}") + + return InfrastructureHealth( + portainer_connected=portainer_healthy, + npm_connected=npm_healthy, + total_stacks=total_stacks, + total_proxy_hosts=total_proxy_hosts + ) + + @router.get( + "/services", + response_model=List[ServiceInfo], + summary="List all deployed services (Docker Compose stacks)", + description="List all services deployed via Portainer stacks. Each 'service' represents a Docker Compose stack." + ) + async def list_services(): + """ + List all deployed Docker Compose stacks from Portainer + + Returns comprehensive stack/service information including: + - Stack name (the service name) + - Status (active/inactive) + - Exposed ports + - Configured domains (from NPM reverse proxy) + - Running container count + """ + portainer = get_portainer_client() + npm = get_npm_client() + + try: + stacks = await portainer.get_stacks() + proxy_hosts = await npm.get_proxy_hosts() + + # Build domain mapping (domain -> service name) + domain_map = {} + for proxy in proxy_hosts: + for domain in proxy.get("domain_names", []): + # Try to extract service name from forward_host + forward_host = proxy.get("forward_host", "") + domain_map[domain] = forward_host + + services = [] + for stack in stacks: + # Find domains for this stack + stack_name = stack.get("Name", "") + endpoint_id = stack.get("EndpointId") + domains = [ + domain for domain, host in domain_map.items() + if stack_name in host or host in stack_name + ] + + # Get container status for this stack + containers_running = 0 + containers_total = 0 + try: + all_containers = await portainer.get_containers(endpoint_id, all_containers=True) + for container in all_containers: + labels = container.get("Labels", {}) + container_stack = labels.get("com.docker.compose.project", "") + + if container_stack.lower() == stack_name.lower(): + containers_total += 1 + if container.get("State", "") == "running": + containers_running += 1 + except Exception as e: + logger.warning(f"Failed to get container status for {stack_name}: {e}") + + service_info = ServiceInfo( + name=stack_name, + stack_id=stack.get("Id"), + status=stack.get("Status", "unknown"), + endpoint_id=endpoint_id, + ports=[], # TODO: Extract from stack file + domains=domains, + running=containers_running > 0, + containers_running=containers_running, + containers_total=containers_total + ) + services.append(service_info) + + return services + + except Exception as e: + logger.error(f"Failed to list services: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.get( + "/services/{name}", + response_model=ServiceInfo, + summary="Get service details" + ) + async def get_service(name: str): + """ + Get detailed information about a specific service + + Args: + name: Service/stack name + + Returns: + Service details including status and configuration + """ + portainer = get_portainer_client() + + try: + stacks = await portainer.get_stacks() + + # Find stack by name (case-insensitive) + stack = next( + (s for s in stacks if s.get("Name", "").lower() == name.lower()), + None + ) + + if not stack: + raise HTTPException(status_code=404, detail=f"Service '{name}' not found") + + return ServiceInfo( + name=stack.get("Name", ""), + stack_id=stack.get("Id"), + status=stack.get("Status", "unknown"), + endpoint_id=stack.get("EndpointId"), + ports=[], + domains=[] + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to get service '{name}': {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post( + "/services/{service}/manage", + response_model=Dict[str, Any], + summary="Manage service lifecycle" + ) + async def manage_service(service: str, action: str, replicas: Optional[int] = None): + """ + Manage Docker Compose service/stack lifecycle. + + Args: + service: Service/stack name + action: One of: "start", "stop", "restart", "scale" + replicas: Number of replicas (required for scale action) + + Returns: + Success status and message + """ + from pydantic import BaseModel + + class ManageServiceRequest(BaseModel): + action: str + replicas: Optional[int] = None + + portainer = get_portainer_client() + logger.info(f"Managing service '{service}': action={action}, replicas={replicas}") + + # Validate action + valid_actions = ["start", "stop", "restart", "scale"] + if action not in valid_actions: + raise HTTPException( + status_code=400, + detail=f"Invalid action '{action}'. Must be one of: {', '.join(valid_actions)}" + ) + + # Validate scale has replicas + if action == "scale" and replicas is None: + raise HTTPException( + status_code=400, + detail="'scale' action requires 'replicas' parameter" + ) + + try: + # Find the stack by name + stacks = await portainer.get_stacks() + stack = next( + (s for s in stacks if s.get("Name", "").lower() == service.lower()), + None + ) + + if not stack: + raise HTTPException( + status_code=404, + detail=f"Service '{service}' not found" + ) + + stack_id = stack.get("Id") + endpoint_id = stack.get("EndpointId") + + # Perform the action + if action in ["start", "stop", "restart"]: + # These actions need to be implemented + # For now, return not implemented + raise HTTPException( + status_code=501, + detail=f"Action '{action}' not yet implemented for services" + ) + + elif action == "scale": + # Scaling requires updating the stack's compose file + raise HTTPException( + status_code=501, + detail="Scaling not yet implemented" + ) + + return { + "success": True, + "action": action, + "service": service, + "message": f"Action '{action}' completed successfully" + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to {action} service '{service}': {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + @router.get( + "/services/{service}/status", + response_model=Dict[str, Any], + summary="Get detailed service status" + ) + async def get_service_status(service: str): + """ + Get comprehensive service status including containers, resources, and events. + + Args: + service: Service/stack name + + Returns: + Detailed status information + """ + portainer = get_portainer_client() + logger.info(f"Getting status for service '{service}'") + + try: + # Find the stack + stacks = await portainer.get_stacks() + stack = next( + (s for s in stacks if s.get("Name", "").lower() == service.lower()), + None + ) + + if not stack: + raise HTTPException( + status_code=404, + detail=f"Service '{service}' not found" + ) + + stack_id = stack.get("Id") + endpoint_id = stack.get("EndpointId") + status_code = stack.get("Status", 0) + + # Get containers for this stack + all_containers = await portainer.list_containers(all_containers=True) + + # Filter containers belonging to this stack + # Stack name is usually in the container labels + stack_containers = [] + for container in all_containers: + labels = container.get('Labels', {}) + # Check if container belongs to this stack + # Docker Compose adds labels like: com.docker.compose.project + project = labels.get('com.docker.compose.project', '').lower() + if project == service.lower(): + stack_containers.append(container) + + # Format container info + container_info = [] + running_count = 0 + for c in stack_containers: + state = c.get('State', 'unknown') + if state == 'running': + running_count += 1 + + container_info.append({ + 'name': c.get('Names', ['unknown'])[0].lstrip('/'), + 'status': state, + 'health': 'N/A', # Would need to inspect container for health + 'uptime': c.get('Status', 'N/A') + }) + + total_containers = len(stack_containers) + replica_status = f"{running_count}/{total_containers} running" + + return { + "name": stack.get("Name"), + "status": "active" if status_code == 1 else "inactive", + "stack_id": stack_id, + "replica_status": replica_status, + "containers": container_info, + "resources": { + "memory_total": "N/A", # Would need to aggregate container stats + "cpu_usage": "N/A" + }, + "recent_events": [] # Would need to query Docker events API + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to get status for service '{service}': {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + @router.get( + "/ports", + response_model=List[PortInfo], + summary="List allocated ports" + ) + async def list_ports(): + """ + List all currently allocated ports + + Scans all running containers to extract: + - Internal and external port mappings + - Container internal hostnames and IPs + - External domain names (from NPM proxy configuration) + """ + portainer = get_portainer_client() + npm = get_npm_client() + + try: + # Get all endpoints (Docker environments) + endpoints = await portainer.get_endpoints() + + # Get proxy hosts for external domain mapping + proxy_hosts = await npm.get_proxy_hosts() + + # Build mapping of forward_host:forward_port -> domains + port_domain_map = {} + for proxy in proxy_hosts: + forward_host = proxy.get("forward_host", "") + forward_port = proxy.get("forward_port", 0) + domains = proxy.get("domain_names", []) + key = f"{forward_host}:{forward_port}" + if key not in port_domain_map: + port_domain_map[key] = [] + port_domain_map[key].extend(domains) + + ports = [] + + # Scan containers on each endpoint + for endpoint in endpoints: + endpoint_id = endpoint.get("Id") + + try: + containers = await portainer.get_containers(endpoint_id, all_containers=False) + + for container in containers: + container_name = container.get("Names", ["unknown"])[0].lstrip("/") + state = container.get("State", "") + + # Skip non-running containers + if state != "running": + continue + + # Extract network information + networks = container.get("NetworkSettings", {}).get("Networks", {}) + internal_hostname = container_name + internal_ip = None + + # Get first network IP + for network_name, network_info in networks.items(): + if network_info.get("IPAddress"): + internal_ip = network_info.get("IPAddress") + break + + # Extract port mappings + port_mappings = container.get("Ports", []) + + for port_mapping in port_mappings: + internal_port = port_mapping.get("PrivatePort") + host_port = port_mapping.get("PublicPort") + protocol = port_mapping.get("Type", "tcp") + + if not internal_port: + continue + + # Find external domains for this port + external_domains = [] + + # Try matching by container name and port + key_by_name = f"{container_name}:{internal_port}" + if key_by_name in port_domain_map: + external_domains.extend(port_domain_map[key_by_name]) + + # Try matching by internal IP and port + if internal_ip: + key_by_ip = f"{internal_ip}:{internal_port}" + if key_by_ip in port_domain_map: + external_domains.extend(port_domain_map[key_by_ip]) + + # Try matching by localhost and host port + if host_port: + for localhost_variant in ["localhost", "127.0.0.1", "192.168.86.149"]: + key_by_host = f"{localhost_variant}:{host_port}" + if key_by_host in port_domain_map: + external_domains.extend(port_domain_map[key_by_host]) + + # Deduplicate external domains + external_domains = list(set(external_domains)) + + # Get service name from stack label or container name + labels = container.get("Labels", {}) + service_name = labels.get("com.docker.compose.service", container_name) + + port_info = PortInfo( + port=internal_port, + service=service_name, + container_name=container_name, + protocol=protocol, + internal_hostname=internal_hostname, + internal_ip=internal_ip, + external_domains=external_domains, + host_port=host_port, + description=f"{container_name} on {endpoint.get('Name', 'unknown')}" + ) + ports.append(port_info) + + except Exception as e: + logger.error(f"Failed to scan containers on endpoint {endpoint_id}: {e}") + continue + + # Deduplicate ports based on (port, container_name, protocol) + seen = set() + unique_ports = [] + for port_info in ports: + key = (port_info.port, port_info.container_name, port_info.protocol) + if key not in seen: + seen.add(key) + unique_ports.append(port_info) + + # Sort by port number + unique_ports.sort(key=lambda p: p.port) + + return unique_ports + + except Exception as e: + logger.error(f"Failed to list ports: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.get( + "/domains", + response_model=List[DomainInfo], + summary="List configured domains" + ) + async def list_domains(): + """ + List all configured domain names + + Returns domain-to-service mappings with SSL status. + """ + npm = get_npm_client() + + try: + proxy_hosts = await npm.get_proxy_hosts() + + domains = [] + for proxy in proxy_hosts: + service_name = proxy.get("forward_host", "localhost") + certificate_id = proxy.get("certificate_id", 0) + + for domain in proxy.get("domain_names", []): + domain_info = DomainInfo( + domain=domain, + service=service_name, + proxy_host_id=proxy.get("id"), + ssl_enabled=certificate_id > 0, + certificate_id=certificate_id if certificate_id > 0 else None + ) + domains.append(domain_info) + + return domains + + except Exception as e: + logger.error(f"Failed to list domains: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + # Write endpoints + @router.post( + "/services", + response_model=OperationResult, + summary="Deploy a new Docker Compose stack", + description="Deploy a new service by creating a Docker Compose stack in Portainer. Provide the stack name and compose file content. Requires admin authentication.", + status_code=201 + ) + async def deploy_service( + request: DeployServiceRequest, + user: Dict = Depends(get_admin_user) + ): + """ + Deploy a new Docker Compose stack via Portainer + + Creates a new Portainer stack from the provided Docker Compose content. + This is equivalent to deploying a stack through the Portainer UI. + + Args: + request: Stack deployment configuration containing: + - name: Stack name (must be unique) + - compose_content: Full docker-compose.yml content as string + - endpoint_id: Portainer endpoint ID (default: 3 for local) + + Returns: + Operation result with created stack details including stack ID + + Raises: + 409: If a stack with the same name already exists + 500: If deployment fails + """ + portainer = get_portainer_client() + + try: + # Check if stack already exists + stacks = await portainer.get_stacks() + existing = next((s for s in stacks if s.get("Name") == request.name), None) + if existing: + raise HTTPException( + status_code=409, + detail=f"Service '{request.name}' already exists with ID {existing.get('Id')}" + ) + + # Create new stack + result = await portainer.create_stack( + name=request.name, + stack_file_content=request.compose_content, + endpoint_id=request.endpoint_id + ) + + logger.info(f"Deployed service '{request.name}' (stack ID: {result.get('Id')})") + + return OperationResult( + success=True, + message=f"Service '{request.name}' deployed successfully", + details=result + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to deploy service '{request.name}': {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.put( + "/services/{name}", + response_model=OperationResult, + summary="Update an existing Docker Compose stack", + description="Update a deployed stack's Docker Compose configuration. This redeploys the stack with the new configuration. Requires admin authentication." + ) + async def update_service( + name: str, + request: UpdateServiceRequest, + user: Dict = Depends(get_admin_user) + ): + """ + Update an existing Docker Compose stack's configuration + + Updates the stack's compose file and redeploys it. This is equivalent + to updating a stack through the Portainer UI. + + Args: + name: Service/stack name + request: Update configuration + + Returns: + Operation result with updated stack details + """ + portainer = get_portainer_client() + + try: + # Find stack by name + stacks = await portainer.get_stacks() + stack = next( + (s for s in stacks if s.get("Name", "").lower() == name.lower()), + None + ) + + if not stack: + raise HTTPException(status_code=404, detail=f"Service '{name}' not found") + + stack_id = stack.get("Id") + endpoint_id = stack.get("EndpointId") + + # Update stack + result = await portainer.update_stack( + stack_id=stack_id, + stack_file_content=request.compose_content, + endpoint_id=endpoint_id, + prune=request.prune, + pull_image=request.pull_image + ) + + logger.info(f"Updated service '{name}' (stack ID: {stack_id})") + + return OperationResult( + success=True, + message=f"Service '{name}' updated successfully", + details=result + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to update service '{name}': {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.delete( + "/services/{name}", + response_model=OperationResult, + summary="Delete a service", + description="Delete a service and remove its stack. Requires admin authentication." + ) + async def delete_service( + name: str, + user: Dict = Depends(get_admin_user) + ): + """ + Delete a service and remove its stack + + Args: + name: Service/stack name + + Returns: + Operation result confirmation + """ + portainer = get_portainer_client() + + try: + # Find stack by name + stacks = await portainer.get_stacks() + stack = next( + (s for s in stacks if s.get("Name", "").lower() == name.lower()), + None + ) + + if not stack: + raise HTTPException(status_code=404, detail=f"Service '{name}' not found") + + stack_id = stack.get("Id") + endpoint_id = stack.get("EndpointId") + + # Delete stack + await portainer.delete_stack( + stack_id=stack_id, + endpoint_id=endpoint_id + ) + + logger.info(f"Deleted service '{name}' (stack ID: {stack_id})") + + return OperationResult( + success=True, + message=f"Service '{name}' deleted successfully", + details={"stack_id": stack_id} + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to delete service '{name}': {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.get( + "/proxy/{proxy_id}", + summary="Get proxy host details" + ) + async def get_proxy_host(proxy_id: int): + """ + Get detailed configuration of a specific proxy host + + Args: + proxy_id: NPM proxy host ID + + Returns: + Complete proxy host configuration including locations + """ + npm = get_npm_client() + + try: + proxy_host = await npm.get_proxy_host(proxy_id) + return proxy_host + except Exception as e: + logger.error(f"Failed to get proxy host {proxy_id}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post( + "/proxy", + response_model=OperationResult, + summary="Create a new proxy host", + description="Create a new Nginx Proxy Manager proxy host with optional SSL certificate. Requires admin authentication.", + status_code=201 + ) + async def create_proxy( + request: CreateProxyRequest, + user: Dict = Depends(get_admin_user) + ): + """ + Create a new Nginx Proxy Manager proxy host + + Optionally request an SSL certificate from Let's Encrypt. + + Args: + request: Proxy host configuration + + Returns: + Operation result with proxy host details + """ + npm = get_npm_client() + + try: + certificate_id = 0 + + # Request SSL certificate if requested + if request.request_ssl_certificate: + logger.info(f"Requesting SSL certificate for {request.domain_names}") + cert_result = await npm.create_certificate( + domain_names=request.domain_names + ) + certificate_id = cert_result.get("id", 0) + logger.info(f"SSL certificate created: ID {certificate_id}") + + # Create proxy host + proxy_result = await npm.create_proxy_host( + domain_names=request.domain_names, + forward_host=request.forward_host, + forward_port=request.forward_port, + forward_scheme=request.forward_scheme, + certificate_id=certificate_id, + ssl_forced=request.ssl_enabled, + block_exploits=request.block_exploits, + websocket_upgrade=request.websocket_upgrade, + http2_support=request.http2_support + ) + + logger.info(f"Created proxy host for {request.domain_names} → {request.forward_host}:{request.forward_port}") + + return OperationResult( + success=True, + message=f"Proxy host created for {', '.join(request.domain_names)}", + details={ + "proxy_host": proxy_result, + "certificate_id": certificate_id if certificate_id > 0 else None + } + ) + + except Exception as e: + logger.error(f"Failed to create proxy host: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.put( + "/proxy/{proxy_id}", + response_model=OperationResult, + summary="Update a proxy host", + description="Update an existing Nginx Proxy Manager proxy host configuration. Requires admin authentication." + ) + async def update_proxy( + proxy_id: int, + config: Dict[str, Any], + user: Dict = Depends(get_admin_user) + ): + """ + Update an existing Nginx Proxy Manager proxy host + + Args: + proxy_id: Proxy host ID to update + config: Full proxy host configuration (get from get_proxy_host, modify, then update) + + Returns: + Operation result with updated proxy host details + """ + npm = get_npm_client() + + try: + result = await npm.update_proxy_host(proxy_id, config) + + logger.info(f"Updated proxy host {proxy_id}: {result.get('domain_names', [])}") + + return OperationResult( + success=True, + message=f"Proxy host {proxy_id} updated successfully", + details={"proxy_host": result} + ) + + except Exception as e: + logger.error(f"Failed to update proxy host {proxy_id}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + # Service Control Endpoints + @router.get( + "/service-groups", + summary="List service groups" + ) + async def list_service_groups(): + """ + List all defined service groups + + Returns service groups with their member services and status. + """ + return { + "groups": service_groups.list_service_groups(), + "always_on": list(service_groups.ALWAYS_ON_SERVICES), + "stoppable": service_groups.list_stoppable_services() + } + + @router.post( + "/services/{name}/stop", + response_model=OperationResult, + summary="Stop a service or service group", + description="Stop a service or service group by pausing monitors and stopping containers. Requires admin authentication when accessed externally via api.schweitz.net." + ) + async def stop_service( + name: str, + user: Dict = Depends(get_forward_auth_admin) + ): + """ + Stop a service or service group + + This will: + 1. Validate service can be stopped (not always-on) + 2. Pause Uptime Kuma monitors for all services in group + 3. Stop the Portainer stack(s) + + Args: + name: Service or group name + + Returns: + Operation result with details + """ + portainer = get_portainer_client() + kuma = get_kuma_client() + + try: + # Get all services in the group + services = service_groups.get_service_group(name) + + # Validate none are always-on + is_valid, error_msg = service_groups.validate_stop_request(services) + if not is_valid: + raise HTTPException(status_code=403, detail=error_msg) + + results = { + "stopped_services": [], + "paused_monitors": [], + "errors": [] + } + + # Stop each service + for service_name in services: + try: + # 1. Pause Uptime Kuma monitor + try: + monitor_paused = await kuma.pause_monitor_by_name(service_name) + if monitor_paused: + results["paused_monitors"].append(service_name) + logger.info(f"Paused Kuma monitor for {service_name}") + except Exception as e: + logger.warning(f"Failed to pause Kuma monitor for {service_name}: {e}") + results["errors"].append(f"Kuma pause failed for {service_name}: {str(e)}") + + # 2. Stop Portainer stack + stacks = await portainer.get_stacks() + stack = next( + (s for s in stacks if s.get("Name", "").lower() == service_name.lower()), + None + ) + + if stack: + stack_id = stack.get("Id") + endpoint_id = stack.get("EndpointId") + + # Stop stack by deleting it (Portainer doesn't have a "stop" operation) + # Note: This is destructive. For a gentler approach, we'd need to use docker compose stop + # Let's use docker API instead + logger.info(f"Stopping containers for stack: {service_name}") + + # Get containers for this stack + containers = await portainer.get_containers(endpoint_id, all_containers=False) + stopped_containers = [] + + for container in containers: + labels = container.get("Labels", {}) + container_stack = labels.get("com.docker.compose.project", "") + + if container_stack.lower() == service_name.lower(): + container_id = container.get("Id") + # Stop container via Portainer Docker API + await portainer.stop_container(endpoint_id, container_id) + stopped_containers.append(container.get("Names", ["unknown"])[0]) + + results["stopped_services"].append({ + "service": service_name, + "stack_id": stack_id, + "containers": stopped_containers + }) + logger.info(f"Stopped service: {service_name}") + else: + results["errors"].append(f"Stack not found: {service_name}") + + except Exception as e: + logger.error(f"Failed to stop service {service_name}: {e}") + results["errors"].append(f"{service_name}: {str(e)}") + + success = len(results["stopped_services"]) > 0 + message = f"Stopped {len(results['stopped_services'])} service(s)" + if results["errors"]: + message += f" with {len(results['errors'])} error(s)" + + return OperationResult( + success=success, + message=message, + details=results + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to stop service group '{name}': {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post( + "/services/{name}/start", + response_model=OperationResult, + summary="Start a service or service group", + description="Start a service or service group by starting containers and resuming monitors. Requires admin authentication when accessed externally via api.schweitz.net." + ) + async def start_service( + name: str, + user: Dict = Depends(get_forward_auth_admin) + ): + """ + Start a service or service group + + This will: + 1. Start the Portainer stack(s) + 2. Resume Uptime Kuma monitors for all services in group + + Args: + name: Service or group name + + Returns: + Operation result with details + """ + portainer = get_portainer_client() + kuma = get_kuma_client() + + try: + # Get all services in the group + services = service_groups.get_service_group(name) + + results = { + "started_services": [], + "resumed_monitors": [], + "errors": [] + } + + # Start each service + for service_name in services: + try: + # 1. Start Portainer stack (start containers) + stacks = await portainer.get_stacks() + stack = next( + (s for s in stacks if s.get("Name", "").lower() == service_name.lower()), + None + ) + + if stack: + stack_id = stack.get("Id") + endpoint_id = stack.get("EndpointId") + + logger.info(f"Starting containers for stack: {service_name}") + + # Get containers for this stack + containers = await portainer.get_containers(endpoint_id, all_containers=True) + started_containers = [] + + for container in containers: + labels = container.get("Labels", {}) + container_stack = labels.get("com.docker.compose.project", "") + + if container_stack.lower() == service_name.lower(): + container_id = container.get("Id") + # Start container via Portainer Docker API + await portainer.start_container(endpoint_id, container_id) + started_containers.append(container.get("Names", ["unknown"])[0]) + + results["started_services"].append({ + "service": service_name, + "stack_id": stack_id, + "containers": started_containers + }) + logger.info(f"Started service: {service_name}") + + # 2. Resume Uptime Kuma monitor + try: + monitor_resumed = await kuma.resume_monitor_by_name(service_name) + if monitor_resumed: + results["resumed_monitors"].append(service_name) + logger.info(f"Resumed Kuma monitor for {service_name}") + except Exception as e: + logger.warning(f"Failed to resume Kuma monitor for {service_name}: {e}") + results["errors"].append(f"Kuma resume failed for {service_name}: {str(e)}") + + else: + results["errors"].append(f"Stack not found: {service_name}") + + except Exception as e: + logger.error(f"Failed to start service {service_name}: {e}") + results["errors"].append(f"{service_name}: {str(e)}") + + success = len(results["started_services"]) > 0 + message = f"Started {len(results['started_services'])} service(s)" + if results["errors"]: + message += f" with {len(results['errors'])} error(s)" + + return OperationResult( + success=success, + message=message, + details=results + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to start service group '{name}': {e}") + raise HTTPException(status_code=500, detail=str(e)) + + # ===== Monitoring Endpoints ===== + + @router.get( + "/widget-data", + summary="Get combined data for service control widget", + response_model=Dict[str, Any] + ) + async def get_widget_data(): + """ + Get combined service and monitor data for the widget + + Returns all data needed by service-control widget in a single call: + - Service list with status and container counts + - Monitor list with uptime percentages + - Service groups and always-on list + + This endpoint is designed for browser-based widgets to avoid + multiple API calls and cross-origin issues. + """ + try: + portainer = get_portainer_client() + kuma = get_kuma_client() + npm = get_npm_client() + + # Fetch services (same logic as /services endpoint) + stacks = await portainer.get_stacks() + proxy_hosts = await npm.get_proxy_hosts() + + # Build domain mapping + domain_map = {} + for proxy in proxy_hosts: + for domain in proxy.get("domain_names", []): + forward_host = proxy.get("forward_host", "") + domain_map[domain] = forward_host + + services = [] + for stack in stacks: + stack_name = stack.get("Name", "") + endpoint_id = stack.get("EndpointId") + domains = [ + domain for domain, host in domain_map.items() + if stack_name in host or host in stack_name + ] + + # Get container status + containers_running = 0 + containers_total = 0 + try: + all_containers = await portainer.get_containers(endpoint_id, all_containers=True) + for container in all_containers: + labels = container.get("Labels", {}) + container_stack = labels.get("com.docker.compose.project", "") + + if container_stack.lower() == stack_name.lower(): + containers_total += 1 + if container.get("State", "") == "running": + containers_running += 1 + except Exception as e: + logger.warning(f"Failed to get container status for {stack_name}: {e}") + + services.append({ + "name": stack_name, + "stack_id": stack.get("Id"), + "status": "active" if stack.get("Status") == 1 else "inactive", + "endpoint_id": endpoint_id, + "domains": domains, + "running": containers_running > 0, + "containers_running": containers_running, + "containers_total": containers_total + }) + + # Fetch monitors with real-time status from metrics endpoint + monitors_list = [] + try: + # Get real-time status from Prometheus metrics + metrics_data = await kuma.get_metrics_status() + + for monitor_name, monitor_info in metrics_data.items(): + # Status: 1=UP, 0=DOWN, 2=PENDING, 3=MAINTENANCE + status = monitor_info.get('status', 0) + + # Convert status to simple up/down for widget + # Treat UP (1) as 100%, anything else as 0% + status_percentage = 100.0 if status == 1 else 0.0 + + monitors_list.append({ + "id": None, # Not available from metrics + "name": monitor_name, + "uptime_24h": status_percentage, # Current status as percentage + "active": True, # Assume active if in metrics + "status": status, # 1=UP, 0=DOWN, 2=PENDING, 3=MAINTENANCE + "response_time": monitor_info.get('response_time', 0) + }) + + logger.info(f"Fetched status for {len(monitors_list)} monitors from metrics") + except Exception as e: + logger.warning(f"Failed to fetch monitors: {e}") + # Continue without monitor data rather than failing + + return { + "success": True, + "services": services, + "monitors": monitors_list, + "service_groups": { + "groups": service_groups.list_service_groups(), + "always_on": list(service_groups.ALWAYS_ON_SERVICES), + "stoppable": service_groups.list_stoppable_services() + } + } + + except Exception as e: + logger.error(f"Failed to fetch widget data: {e}") + raise HTTPException(status_code=500, detail=f"Failed to fetch widget data: {str(e)}") + + @router.get( + "/monitors", + summary="List all monitors", + response_model=Dict[str, Any] + ) + async def list_monitors(): + """ + List all Uptime Kuma monitors + + Returns: + List of monitors with their configurations + """ + try: + kuma = get_kuma_client() + monitors = await kuma.get_monitors() + + return { + "success": True, + "monitors": monitors, + "total": len(monitors) + } + + except Exception as e: + logger.error(f"Failed to list monitors: {e}") + raise HTTPException(status_code=500, detail=f"Failed to list monitors: {str(e)}") + + @router.post( + "/monitors", + summary="Create a new monitor", + description="Create a new Uptime Kuma monitor. Requires admin authentication.", + response_model=Dict[str, Any] + ) + async def create_monitor( + monitor_config: Dict[str, Any], + user: Dict = Depends(get_admin_user) + ): + """ + Create a new Uptime Kuma monitor + + Args: + monitor_config: Monitor configuration (name, type, hostname, port, etc.) + + Returns: + Created monitor details including ID + """ + try: + kuma = get_kuma_client() + created_monitor = await kuma.add_monitor(monitor_config) + + return { + "success": True, + "message": f"Monitor '{monitor_config.get('name')}' created successfully", + "monitor": created_monitor + } + + except Exception as e: + logger.error(f"Failed to create monitor: {e}") + raise HTTPException(status_code=500, detail=f"Failed to create monitor: {str(e)}") + + @router.get( + "/monitors/{monitor_id}", + summary="Get monitor details", + response_model=Dict[str, Any] + ) + async def get_monitor(monitor_id: int): + """ + Get details of a specific monitor + + Args: + monitor_id: Monitor identifier + + Returns: + Monitor configuration and status + """ + try: + kuma = get_kuma_client() + monitor = await kuma.get_monitor(monitor_id) + + return { + "success": True, + "monitor": monitor + } + + except Exception as e: + logger.error(f"Failed to get monitor {monitor_id}: {e}") + raise HTTPException(status_code=404, detail=f"Monitor {monitor_id} not found: {str(e)}") + + @router.put( + "/monitors/{monitor_id}", + summary="Update a monitor", + description="Update an existing Uptime Kuma monitor. Requires admin authentication.", + response_model=Dict[str, Any] + ) + async def update_monitor( + monitor_id: int, + updates: Dict[str, Any], + user: Dict = Depends(get_admin_user) + ): + """ + Update an existing monitor + + Args: + monitor_id: Monitor identifier + updates: Fields to update + + Returns: + Updated monitor details + """ + try: + kuma = get_kuma_client() + + # Get existing monitor + existing = await kuma.get_monitor(monitor_id) + + # Merge updates + monitor_config = existing.copy() + monitor_config.update(updates) + + # Update monitor + updated_monitor = await kuma.update_monitor(monitor_id, monitor_config) + + return { + "success": True, + "message": f"Monitor {monitor_id} updated successfully", + "monitor": updated_monitor + } + + except Exception as e: + logger.error(f"Failed to update monitor {monitor_id}: {e}") + raise HTTPException(status_code=500, detail=f"Failed to update monitor: {str(e)}") + + @router.delete( + "/monitors/{monitor_id}", + summary="Delete a monitor", + description="Delete an Uptime Kuma monitor. Requires admin authentication.", + response_model=Dict[str, Any] + ) + async def delete_monitor( + monitor_id: int, + user: Dict = Depends(get_admin_user) + ): + """ + Delete a monitor + + Args: + monitor_id: Monitor identifier + + Returns: + Success confirmation + """ + try: + kuma = get_kuma_client() + await kuma.delete_monitor(monitor_id) + + return { + "success": True, + "message": f"Monitor {monitor_id} deleted successfully" + } + + except Exception as e: + logger.error(f"Failed to delete monitor {monitor_id}: {e}") + raise HTTPException(status_code=500, detail=f"Failed to delete monitor: {str(e)}") + + # ======================================================================== + # Container Management Endpoints (for core-ai infrastructure tools) + # ======================================================================== + + @router.get( + "/containers", + response_model=List[Dict[str, Any]], + summary="List Docker containers" + ) + async def list_containers(status: Optional[str] = "running"): + """ + List Docker containers with optional status filter. + + Args: + status: Filter by status - "all", "running", "stopped", "paused" (default: "running") + + Returns: + List of containers in Docker API format + """ + portainer = get_portainer_client() + logger.info(f"Listing containers (status filter: {status})") + + try: + # Get all containers using Portainer with Docker socket fallback + all_containers_flag = status in ["all", "stopped"] + containers = await portainer.list_containers(all_containers=all_containers_flag) + + # Filter by status if needed + if status == "running": + containers = [c for c in containers if c.get('State') == 'running'] + elif status == "stopped": + containers = [c for c in containers if c.get('State') != 'running'] + elif status == "paused": + containers = [c for c in containers if c.get('State') == 'paused'] + + logger.info(f"Found {len(containers)} containers matching status '{status}'") + return containers + + except Exception as e: + logger.error(f"Failed to list containers: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to list containers: {str(e)}") + + @router.post( + "/containers/{container}/{action}", + response_model=Dict[str, Any], + summary="Manage container state" + ) + async def manage_container(container: str, action: str): + """ + Perform lifecycle operation on a Docker container. + + Args: + container: Container name or ID + action: One of: "start", "stop", "restart", "pause", "unpause", "remove" + + Returns: + Success status and message + """ + portainer = get_portainer_client() + logger.info(f"Managing container '{container}': action={action}") + + # Validate action + valid_actions = ["start", "stop", "restart", "pause", "unpause", "remove"] + if action not in valid_actions: + raise HTTPException( + status_code=400, + detail=f"Invalid action '{action}'. Must be one of: {', '.join(valid_actions)}" + ) + + try: + # Find the container + all_containers = await portainer.list_containers(all_containers=True) + + matching_container = None + for c in all_containers: + names = c.get('Names', []) + for name in names: + clean_name = name.lstrip('/') + if clean_name.lower() == container.lower(): + matching_container = c + break + if matching_container: + break + + if not matching_container: + raise HTTPException( + status_code=404, + detail=f"Container '{container}' not found" + ) + + container_id = matching_container['Id'] + + # Get endpoints + endpoints = await portainer.get_endpoints() + if not endpoints: + raise HTTPException(status_code=500, detail="No Portainer endpoints available") + + endpoint_id = endpoints[0]['Id'] + + # Perform the action + if action == "start": + await portainer.start_container(endpoint_id, container_id) + message = f"Started container '{container}' successfully" + + elif action == "stop": + await portainer.stop_container(endpoint_id, container_id) + message = f"Stopped container '{container}' successfully" + + elif action == "restart": + await portainer.stop_container(endpoint_id, container_id) + await portainer.start_container(endpoint_id, container_id) + message = f"Restarted container '{container}' successfully" + + elif action in ["pause", "unpause", "remove"]: + # These actions need to be added to Portainer client + # For now, return error + raise HTTPException( + status_code=501, + detail=f"Action '{action}' not yet implemented" + ) + + logger.info(f"Successfully {action}ed container '{container}'") + + return { + "success": True, + "action": action, + "container": container, + "message": message + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to {action} container '{container}': {e}", exc_info=True) + + # Provide helpful error messages + error_str = str(e).lower() + if "304" in error_str or "not modified" in error_str: + raise HTTPException( + status_code=304, + detail=f"Container '{container}' is already in the target state" + ) + elif "conflict" in error_str: + raise HTTPException( + status_code=409, + detail=f"Cannot {action} container '{container}': state conflict" + ) + else: + raise HTTPException(status_code=500, detail=str(e)) + + @router.get( + "/containers/{container}", + response_model=Dict[str, Any], + summary="Inspect container" + ) + async def inspect_container( + container: str, + details: Optional[str] = "summary" + ): + """ + Get detailed information about a Docker container. + + Args: + container: Container name or ID + details: Level of detail - "summary", "full", or "resources" (not used server-side, for client formatting) + + Returns: + Container details in Docker inspect format + """ + portainer = get_portainer_client() + logger.info(f"Inspecting container '{container}' (details={details})") + + try: + # Use Portainer's inspect_container which auto-detects endpoint and falls back to Docker socket + info = await portainer.inspect_container(container) + + if not info: + raise HTTPException( + status_code=404, + detail=f"Container '{container}' not found" + ) + + logger.info(f"Successfully inspected container '{container}'") + return info + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to inspect container '{container}': {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + @router.get( + "/containers/{container}/logs", + response_model=Dict[str, Any], + summary="Get container logs" + ) + async def get_container_logs( + container: str, + lines: Optional[int] = 50, + since: Optional[str] = None + ): + """ + Retrieve logs from a Docker container. + + Args: + container: Container name or ID + lines: Number of log lines to retrieve (default: 50, max: 500) + since: Time filter - "1h", "30m", or ISO timestamp (not yet implemented) + + Returns: + Container logs with metadata + """ + import httpx + + logger.info(f"Retrieving logs for container '{container}' (lines={lines}, since={since})") + + # Clamp lines to reasonable limit + lines = min(max(1, lines), 500) + + try: + # Access Docker socket directly to get logs + transport = httpx.AsyncHTTPTransport(uds="/var/run/docker.sock") + async with httpx.AsyncClient(transport=transport, timeout=15.0) as client: + # Get container logs via Docker API + params = { + "stdout": "true", + "stderr": "true", + "tail": lines, + "timestamps": "true" + } + + # TODO: Add 'since' parameter support + # if since: + # params["since"] = parse_since_to_timestamp(since) + + response = await client.get( + f"http://localhost/v1.41/containers/{container}/logs", + params=params + ) + + if response.status_code == 404: + raise HTTPException( + status_code=404, + detail=f"Container '{container}' not found" + ) + + response.raise_for_status() + logs_raw = response.text + + # Docker logs come with binary prefixes (8 bytes per line) + # Strip these headers for cleaner output + lines_list = logs_raw.split('\n') + cleaned_lines = [] + + for line in lines_list: + if len(line) > 8: + # Skip the 8-byte Docker stream header if present + cleaned_line = line[8:] if line[0:1] in [b'\x01', b'\x02', '\x01', '\x02'] else line + cleaned_lines.append(cleaned_line) + elif line: + cleaned_lines.append(line) + + logs = '\n'.join(cleaned_lines).strip() + + logger.info(f"Successfully retrieved logs for container '{container}' ({len(cleaned_lines)} lines)") + + return { + "container": container, + "lines_requested": lines, + "since": since, + "logs": logs + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to retrieve logs for container '{container}': {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + # ======================================================================== + # Monitoring & Resource Endpoints (for core-ai infrastructure tools) + # ======================================================================== + + @router.get( + "/resources/system", + response_model=Dict[str, Any], + summary="Get system resource usage" + ) + async def get_system_resources(): + """ + Get overall system resource usage. + + Returns: + System-level metrics including CPU, memory, disk, and network + """ + logger.info("Getting system resources") + + try: + # Access Docker system info via socket + import httpx + + transport = httpx.AsyncHTTPTransport(uds="/var/run/docker.sock") + async with httpx.AsyncClient(transport=transport, timeout=10.0) as client: + # Get system info + info_response = await client.get("http://localhost/v1.41/info") + info_response.raise_for_status() + info = info_response.json() + + # Get system df (disk usage) + df_response = await client.get("http://localhost/v1.41/system/df") + df_response.raise_for_status() + df = df_response.json() + + # Extract system metrics + ncpu = info.get('NCPU', 0) + mem_total = info.get('MemTotal', 0) + + # Calculate memory usage (rough estimate) + # Docker doesn't directly provide used memory, so we estimate + mem_used = mem_total * 0.5 # Placeholder - would need psutil or /proc + mem_available = mem_total - mem_used + mem_usage_pct = (mem_used / mem_total * 100) if mem_total > 0 else 0 + + # Disk usage from system/df + layers_size = sum(img.get('Size', 0) for img in df.get('Images', [])) + containers_size = sum(c.get('SizeRw', 0) for c in df.get('Containers', [])) + volumes_size = sum(v.get('UsageData', {}).get('Size', 0) for v in df.get('Volumes', [])) + + total_disk_used = layers_size + containers_size + volumes_size + + return { + "cpu": { + "cores": ncpu, + "usage_percent": None, # Would need to calculate from stats over time + "load_average": [] # Not available from Docker API + }, + "memory": { + "total_bytes": mem_total, + "used_bytes": mem_used, + "available_bytes": mem_available, + "usage_percent": mem_usage_pct + }, + "disk": { + "total_bytes": None, # Not available from Docker API + "used_bytes": total_disk_used, + "available_bytes": None, + "usage_percent": None + }, + "network": { + "interfaces": {} # Would need to parse network stats + } + } + + except Exception as e: + logger.error(f"Failed to get system resources: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + @router.get( + "/resources/containers", + response_model=List[Dict[str, Any]], + summary="Get container resource usage" + ) + async def get_container_resources(container: Optional[str] = None): + """ + Get container-specific resource usage. + + Args: + container: Optional specific container name/ID + + Returns: + List of container resource stats + """ + logger.info(f"Getting container resources (container={container})") + + try: + import httpx + portainer = get_portainer_client() + + # Get list of containers + if container: + # Get specific container + all_containers = await portainer.list_containers(all_containers=False) + containers = [c for c in all_containers + if container.lower() in c.get('Names', [''])[0].lower()] + + if not containers: + raise HTTPException(status_code=404, detail=f"Container '{container}' not found") + else: + # Get all running containers + containers = await portainer.list_containers(all_containers=False) + + # Get stats for each container + stats_list = [] + transport = httpx.AsyncHTTPTransport(uds="/var/run/docker.sock") + + async with httpx.AsyncClient(transport=transport, timeout=15.0) as client: + for c in containers: + container_id = c.get('Id') + name = c.get('Names', ['unknown'])[0].lstrip('/') + + try: + # Get container stats (one-time, not streaming) + stats_response = await client.get( + f"http://localhost/v1.41/containers/{container_id}/stats", + params={"stream": "false"} + ) + stats_response.raise_for_status() + stats = stats_response.json() + + # Parse stats + cpu_stats = stats.get('cpu_stats', {}) + precpu_stats = stats.get('precpu_stats', {}) + memory_stats = stats.get('memory_stats', {}) + networks = stats.get('networks', {}) + blkio_stats = stats.get('blkio_stats', {}) + + # Calculate CPU percentage + cpu_delta = cpu_stats.get('cpu_usage', {}).get('total_usage', 0) - \ + precpu_stats.get('cpu_usage', {}).get('total_usage', 0) + system_delta = cpu_stats.get('system_cpu_usage', 0) - \ + precpu_stats.get('system_cpu_usage', 0) + online_cpus = cpu_stats.get('online_cpus', 1) + + cpu_percent = 0.0 + if system_delta > 0 and cpu_delta > 0: + cpu_percent = (cpu_delta / system_delta) * online_cpus * 100.0 + + # Memory stats + mem_usage = memory_stats.get('usage', 0) + mem_limit = memory_stats.get('limit', 0) + mem_percent = (mem_usage / mem_limit * 100) if mem_limit > 0 else 0 + + # Network stats + net_rx = sum(net.get('rx_bytes', 0) for net in networks.values()) + net_tx = sum(net.get('tx_bytes', 0) for net in networks.values()) + + # Block I/O stats + io_service_bytes = blkio_stats.get('io_service_bytes_recursive', []) + block_read = sum(entry.get('value', 0) for entry in io_service_bytes + if entry.get('op') == 'Read') + block_write = sum(entry.get('value', 0) for entry in io_service_bytes + if entry.get('op') == 'Write') + + stats_list.append({ + "name": name, + "cpu_percent": cpu_percent, + "memory_usage_bytes": mem_usage, + "memory_limit_bytes": mem_limit, + "memory_percent": mem_percent, + "network_rx_bytes": net_rx, + "network_tx_bytes": net_tx, + "block_read_bytes": block_read, + "block_write_bytes": block_write + }) + + except Exception as e: + logger.warning(f"Failed to get stats for container {name}: {e}") + # Skip containers that fail to get stats + continue + + return stats_list + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to get container resources: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + return router + + +# Create controller instance +infrastructure_controller = InfrastructureController() diff --git a/src/controllers/static_controller.py b/src/controllers/static_controller.py new file mode 100644 index 0000000..e1cef91 --- /dev/null +++ b/src/controllers/static_controller.py @@ -0,0 +1,116 @@ +""" +Static Files Controller + +Serves static files for widgets and other frontend assets. +""" +from fastapi import APIRouter +from fastapi.responses import FileResponse, HTMLResponse +from pathlib import Path +import os + +from src.controllers.base import BaseController +from src.logging_config import get_logger + +logger = get_logger(__name__) + + +class StaticController(BaseController): + """ + Controller for serving static files + + Provides endpoints for: + - Organizr widgets + - Other static assets + """ + + def __init__(self): + super().__init__(prefix="/static", tags=["Static"]) + self.static_dir = Path(__file__).parent.parent.parent / "static" + + def create_router(self) -> APIRouter: + """Create and configure the router""" + router = APIRouter(prefix=self.prefix, tags=self.tags) + + @router.get( + "/widgets/{filename}", + response_class=HTMLResponse, + summary="Get widget file" + ) + async def get_widget(filename: str): + """ + Serve widget HTML files + + Args: + filename: Widget filename (e.g., service-control.html) + + Returns: + HTML file content + """ + widget_path = self.static_dir / "widgets" / filename + + if not widget_path.exists(): + return HTMLResponse( + content=f"

404 - Widget not found

{filename}

", + status_code=404 + ) + + if not widget_path.is_file(): + return HTMLResponse( + content=f"

400 - Not a file

", + status_code=400 + ) + + # Security: Ensure the path is within the static directory + try: + widget_path.resolve().relative_to(self.static_dir.resolve()) + except ValueError: + return HTMLResponse( + content=f"

403 - Forbidden

", + status_code=403 + ) + + logger.info(f"Serving widget: {filename}") + return FileResponse( + widget_path, + media_type="text/html", + headers={ + "Cache-Control": "no-cache, no-store, must-revalidate", + "Pragma": "no-cache", + "Expires": "0" + } + ) + + @router.get( + "/widgets", + summary="List available widgets" + ) + async def list_widgets(): + """ + List all available widget files + + Returns: + List of widget filenames + """ + widgets_dir = self.static_dir / "widgets" + + if not widgets_dir.exists(): + return {"widgets": [], "message": "Widgets directory not found"} + + widgets = [] + for file in widgets_dir.glob("*.html"): + widgets.append({ + "name": file.name, + "url": f"/static/widgets/{file.name}", + "size": file.stat().st_size + }) + + return { + "widgets": widgets, + "count": len(widgets) + } + + return router + + +# Create controller instance +static_controller = StaticController() diff --git a/src/controllers/tools_controller.py b/src/controllers/tools_controller.py new file mode 100644 index 0000000..7313772 --- /dev/null +++ b/src/controllers/tools_controller.py @@ -0,0 +1,168 @@ +""" +Tools Controller + +Provides utility tool endpoints including: +- Web scraping and content extraction +- DNS lookups +""" +from fastapi import APIRouter, HTTPException, status + +from src.controllers.base import BaseController +from src.logging_config import get_logger +from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse +from src.web_scraper.service import WebScraperService +from src.web_scraper.exceptions import FetchError, ScrapingError +from src.dns.schemas import DNSLookupRequest, DNSLookupResponse +from src.dns.service import DNSService +from src.dns.exceptions import DNSQueryError + +logger = get_logger(__name__) + + +class ToolsController(BaseController): + """ + Controller for utility tools + + Provides endpoints for: + - Web scraping and content extraction + - DNS lookups + """ + + def __init__(self): + super().__init__(prefix="/tools", tags=["Tools"]) + # Initialize services (could be dependency injected for testing) + self.scraper_service = WebScraperService() + self.dns_service = DNSService() + + def create_router(self) -> APIRouter: + """Create and configure the router""" + router = APIRouter(prefix=self.prefix, tags=self.tags) + + @router.post( + "/scrape", + response_model=WebScraperResponse, + status_code=status.HTTP_200_OK, + summary="Scrape website content", + description=""" + Scrape and extract main content from a website. + + Uses trafilatura for intelligent content extraction (articles, blog posts, documentation), + with BeautifulSoup as fallback. Perfect for feeding webpage content to LLMs. + + **Features:** + - Intelligent main content extraction + - Removes navigation, ads, footers + - Optional link extraction + - Configurable content length limits + + **Rate Limiting:** None (internal network use only) + """ + ) + async def scrape_website(request: WebScraperRequest) -> WebScraperResponse: + """ + Scrape a website and extract its main content + + Args: + request: Scraping request with URL and options + + Returns: + Extracted content with metadata + + Raises: + HTTPException: 400 for fetch errors, 500 for processing errors + """ + try: + logger.info(f"Received scrape request for: {request.url}") + result = await self.scraper_service.scrape_url(request) + return result + + except FetchError as e: + logger.warning(f"Fetch failed: {str(e)}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Failed to fetch URL: {str(e)}" + ) + + except ScrapingError as e: + logger.error(f"Scraping failed: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to extract content: {str(e)}" + ) + + except Exception as e: + logger.error(f"Unexpected error: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="An unexpected error occurred" + ) + + @router.post( + "/dns/lookup", + response_model=DNSLookupResponse, + status_code=status.HTTP_200_OK, + summary="Perform DNS lookup", + description=""" + Perform DNS lookups for various record types. + + Uses dnspython for reliable DNS queries with support for multiple record types + and custom nameservers. Perfect for troubleshooting DNS issues and checking + domain configurations. + + **Supported Record Types:** + - A: IPv4 address records + - AAAA: IPv6 address records + - MX: Mail exchange records + - TXT: Text records (SPF, DKIM, etc.) + - CNAME: Canonical name records + - NS: Nameserver records + - SOA: Start of authority records + - PTR: Pointer records (reverse DNS) + - CAA: Certification authority authorization + - SRV: Service records + + **Features:** + - Custom nameserver support (e.g., 8.8.8.8, 1.1.1.1) + - Query time measurement + - Detailed error messages + + **Rate Limiting:** None (internal network use only) + """ + ) + async def dns_lookup(request: DNSLookupRequest) -> DNSLookupResponse: + """ + Perform DNS lookup for a domain + + Args: + request: DNS lookup request with domain, record type, and optional nameserver + + Returns: + DNS lookup results with records and metadata + + Raises: + HTTPException: 400 for invalid queries, 500 for processing errors + """ + try: + logger.info(f"Received DNS lookup request for: {request.domain} ({request.record_type})") + result = await self.dns_service.lookup(request) + return result + + except DNSQueryError as e: + logger.warning(f"DNS query error: {str(e)}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"DNS query failed: {str(e)}" + ) + + except Exception as e: + logger.error(f"Unexpected error during DNS lookup: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="An unexpected error occurred during DNS lookup" + ) + + return router + + +# Create controller instance +tools_controller = ToolsController() diff --git a/src/credentials.example.py b/src/credentials.example.py new file mode 100644 index 0000000..c2a152e --- /dev/null +++ b/src/credentials.example.py @@ -0,0 +1,24 @@ +""" +Infrastructure Credentials Template + +INSTRUCTIONS: +1. Copy this file to credentials.py +2. Fill in your actual credentials +3. DO NOT commit credentials.py to version control (it's in .gitignore) + +This file should be committed to the repository as a template. +""" + +# Portainer Configuration +PORTAINER_URL = "http://localhost:8001" +PORTAINER_API_KEY = "ptr_your_api_token_here" # Create in Portainer UI: User menu → My account → Access tokens + +# Nginx Proxy Manager Configuration +NPM_URL = "http://localhost:81" +NPM_EMAIL = "admin@example.com" +NPM_PASSWORD = "your_password_here" + +# Uptime Kuma Configuration +KUMA_URL = "http://localhost:3001" +KUMA_USERNAME = "admin" +KUMA_PASSWORD = "your_password_here" diff --git a/src/dns/__init__.py b/src/dns/__init__.py new file mode 100644 index 0000000..f0299fb --- /dev/null +++ b/src/dns/__init__.py @@ -0,0 +1,16 @@ +""" +DNS lookup module + +Provides DNS query functionality for the Core API. +""" +from src.dns.service import DNSService +from src.dns.schemas import DNSLookupRequest, DNSLookupResponse, DNSRecord +from src.dns.exceptions import DNSQueryError + +__all__ = [ + "DNSService", + "DNSLookupRequest", + "DNSLookupResponse", + "DNSRecord", + "DNSQueryError", +] diff --git a/src/dns/exceptions.py b/src/dns/exceptions.py new file mode 100644 index 0000000..f59b840 --- /dev/null +++ b/src/dns/exceptions.py @@ -0,0 +1,8 @@ +""" +DNS-specific exceptions +""" + + +class DNSQueryError(Exception): + """Raised when a DNS query fails""" + pass diff --git a/src/dns/schemas.py b/src/dns/schemas.py new file mode 100644 index 0000000..a723170 --- /dev/null +++ b/src/dns/schemas.py @@ -0,0 +1,94 @@ +""" +Pydantic schemas for DNS lookup module +""" +from pydantic import Field +from typing import Optional, List +from datetime import datetime +from src.base_schema import BaseSchema + + +class DNSLookupRequest(BaseSchema): + """Request model for DNS lookup""" + + domain: str = Field( + ..., + description="The domain name to lookup", + examples=["example.com", "google.com"], + min_length=1, + max_length=255 + ) + + record_type: str = Field( + default="A", + description="DNS record type to query (A, AAAA, MX, TXT, CNAME, NS, SOA, PTR, CAA)", + examples=["A", "AAAA", "MX", "TXT", "CNAME"] + ) + + nameserver: Optional[str] = Field( + default=None, + description="Optional nameserver to use for the query (e.g., 8.8.8.8, 1.1.1.1)", + examples=["8.8.8.8", "1.1.1.1", "9.9.9.9"] + ) + + +class DNSRecord(BaseSchema): + """Single DNS record result""" + + value: str = Field( + ..., + description="The DNS record value" + ) + + ttl: Optional[int] = Field( + default=None, + description="Time to live in seconds" + ) + + priority: Optional[int] = Field( + default=None, + description="Priority (for MX records)" + ) + + +class DNSLookupResponse(BaseSchema): + """Response model for DNS lookup""" + + domain: str = Field( + ..., + description="The queried domain name" + ) + + record_type: str = Field( + ..., + description="DNS record type queried" + ) + + records: List[DNSRecord] = Field( + ..., + description="List of DNS records found" + ) + + nameserver_used: Optional[str] = Field( + default=None, + description="Nameserver used for the query" + ) + + query_time_ms: float = Field( + ..., + description="Query execution time in milliseconds" + ) + + queried_at: datetime = Field( + ..., + description="UTC timestamp when query was executed" + ) + + success: bool = Field( + ..., + description="Whether the query was successful" + ) + + error_message: Optional[str] = Field( + default=None, + description="Error message if query failed" + ) diff --git a/src/dns/service.py b/src/dns/service.py new file mode 100644 index 0000000..84e732f --- /dev/null +++ b/src/dns/service.py @@ -0,0 +1,216 @@ +""" +DNS Lookup Service + +Provides DNS query functionality using dnspython library. +""" +import time +from datetime import datetime, timezone +from typing import Optional + +import dns.resolver +import dns.exception + +from src.logging_config import get_logger +from src.dns.schemas import DNSLookupRequest, DNSLookupResponse, DNSRecord +from src.dns.exceptions import DNSQueryError + +logger = get_logger(__name__) + + +class DNSService: + """ + Service for performing DNS lookups + + Uses dnspython for reliable DNS queries with support for + various record types and custom nameservers. + """ + + # Supported record types + SUPPORTED_RECORD_TYPES = [ + "A", "AAAA", "MX", "TXT", "CNAME", "NS", "SOA", "PTR", "CAA", "SRV" + ] + + def __init__(self): + """Initialize DNS service""" + self.resolver = dns.resolver.Resolver() + # Set reasonable timeout + self.resolver.timeout = 5.0 + self.resolver.lifetime = 10.0 + + async def lookup(self, request: DNSLookupRequest) -> DNSLookupResponse: + """ + Perform DNS lookup for the specified domain and record type + + Args: + request: DNS lookup request with domain, record type, and optional nameserver + + Returns: + DNSLookupResponse with query results + + Raises: + DNSQueryError: If the DNS query fails + """ + start_time = time.time() + record_type = request.record_type.upper() + + # Validate record type + if record_type not in self.SUPPORTED_RECORD_TYPES: + raise DNSQueryError( + f"Unsupported record type: {record_type}. " + f"Supported types: {', '.join(self.SUPPORTED_RECORD_TYPES)}" + ) + + # Configure nameserver if specified + resolver = dns.resolver.Resolver() + resolver.timeout = 5.0 + resolver.lifetime = 10.0 + + nameserver_used = None + if request.nameserver: + resolver.nameservers = [request.nameserver] + nameserver_used = request.nameserver + logger.info(f"Using custom nameserver: {request.nameserver}") + else: + nameserver_used = resolver.nameservers[0] if resolver.nameservers else "system" + + try: + logger.info(f"Performing DNS lookup: {request.domain} ({record_type})") + + # Perform the DNS query + answers = resolver.resolve(request.domain, record_type) + + # Parse results + records = [] + for rdata in answers: + record = self._parse_record(rdata, record_type) + if record: + records.append(record) + + query_time_ms = (time.time() - start_time) * 1000 + + logger.info( + f"DNS lookup successful: {request.domain} ({record_type}) - " + f"Found {len(records)} records in {query_time_ms:.2f}ms" + ) + + return DNSLookupResponse( + domain=request.domain, + record_type=record_type, + records=records, + nameserver_used=nameserver_used, + query_time_ms=round(query_time_ms, 2), + queried_at=datetime.now(timezone.utc), + success=True, + error_message=None + ) + + except dns.resolver.NXDOMAIN: + error_msg = f"Domain not found: {request.domain}" + logger.warning(error_msg) + return self._error_response(request, nameserver_used, start_time, error_msg) + + except dns.resolver.NoAnswer: + error_msg = f"No {record_type} records found for {request.domain}" + logger.warning(error_msg) + return self._error_response(request, nameserver_used, start_time, error_msg) + + except dns.resolver.Timeout: + error_msg = f"DNS query timeout for {request.domain}" + logger.error(error_msg) + return self._error_response(request, nameserver_used, start_time, error_msg) + + except dns.exception.DNSException as e: + error_msg = f"DNS error: {str(e)}" + logger.error(f"DNS query failed for {request.domain}: {e}") + return self._error_response(request, nameserver_used, start_time, error_msg) + + except Exception as e: + error_msg = f"Unexpected error: {str(e)}" + logger.error(f"Unexpected error during DNS lookup: {e}", exc_info=True) + return self._error_response(request, nameserver_used, start_time, error_msg) + + def _parse_record(self, rdata, record_type: str) -> Optional[DNSRecord]: + """ + Parse DNS record data into DNSRecord schema + + Args: + rdata: DNS record data from dnspython + record_type: Type of DNS record + + Returns: + Parsed DNSRecord or None if parsing fails + """ + try: + if record_type == "A" or record_type == "AAAA": + return DNSRecord(value=str(rdata), ttl=None) + + elif record_type == "MX": + return DNSRecord( + value=str(rdata.exchange), + priority=rdata.preference, + ttl=None + ) + + elif record_type == "TXT": + # TXT records can have multiple strings + txt_value = " ".join([s.decode() if isinstance(s, bytes) else str(s) for s in rdata.strings]) + return DNSRecord(value=txt_value, ttl=None) + + elif record_type in ["CNAME", "NS", "PTR"]: + return DNSRecord(value=str(rdata.target), ttl=None) + + elif record_type == "SOA": + soa_value = f"mname={rdata.mname} rname={rdata.rname} serial={rdata.serial}" + return DNSRecord(value=soa_value, ttl=None) + + elif record_type == "CAA": + caa_value = f"{rdata.flags} {rdata.tag.decode() if isinstance(rdata.tag, bytes) else rdata.tag} {rdata.value.decode() if isinstance(rdata.value, bytes) else rdata.value}" + return DNSRecord(value=caa_value, ttl=None) + + elif record_type == "SRV": + srv_value = f"{rdata.target} port={rdata.port} priority={rdata.priority} weight={rdata.weight}" + return DNSRecord( + value=srv_value, + priority=rdata.priority, + ttl=None + ) + + else: + # Fallback for other record types + return DNSRecord(value=str(rdata), ttl=None) + + except Exception as e: + logger.error(f"Failed to parse {record_type} record: {e}") + return None + + def _error_response( + self, + request: DNSLookupRequest, + nameserver_used: Optional[str], + start_time: float, + error_message: str + ) -> DNSLookupResponse: + """ + Create an error response for failed DNS queries + + Args: + request: Original DNS lookup request + nameserver_used: Nameserver that was used + start_time: Query start time + error_message: Error message to include + + Returns: + DNSLookupResponse with error details + """ + query_time_ms = (time.time() - start_time) * 1000 + + return DNSLookupResponse( + domain=request.domain, + record_type=request.record_type.upper(), + records=[], + nameserver_used=nameserver_used, + query_time_ms=round(query_time_ms, 2), + queried_at=datetime.now(timezone.utc), + success=False, + error_message=error_message + ) diff --git a/src/logging_config.py b/src/logging_config.py new file mode 100644 index 0000000..13a4f44 --- /dev/null +++ b/src/logging_config.py @@ -0,0 +1,49 @@ +""" +Logging configuration for Core Code API +""" +import logging +import sys +from pathlib import Path + + +def setup_logging(log_level: str = "INFO") -> None: + """ + Configure logging for the application + + Args: + log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + """ + # Create logs directory if it doesn't exist + log_dir = Path("logs") + log_dir.mkdir(exist_ok=True) + + # Configure root logger + logging.basicConfig( + level=getattr(logging, log_level.upper()), + format="%(asctime)s | %(levelname)-8s | %(name)s:%(funcName)s:%(lineno)d | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + # Console handler + logging.StreamHandler(sys.stdout), + # File handler + logging.FileHandler(log_dir / "app.log", encoding="utf-8") + ] + ) + + # Set specific log levels for third-party libraries + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpcore").setLevel(logging.WARNING) + logging.getLogger("uvicorn.access").setLevel(logging.WARNING) + + +def get_logger(name: str) -> logging.Logger: + """ + Get a logger instance + + Args: + name: Logger name (typically __name__) + + Returns: + Configured logger instance + """ + return logging.getLogger(name) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..9421594 --- /dev/null +++ b/src/main.py @@ -0,0 +1,177 @@ +""" +Main FastAPI application for Core Code API +""" +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from contextlib import asynccontextmanager + +from src.config import get_settings +from src.logging_config import setup_logging, get_logger +from src.models.ollama_client import get_ollama_client, close_ollama_client +from src.controllers.infrastructure_controller import infrastructure_controller +from src.controllers.tools_controller import tools_controller +from src.controllers.health_controller import health_controller +from src.controllers.static_controller import static_controller +from src.controllers.ai_controller import router as ai_router +from src.security import initialize_oidc + +# Initialize settings +settings = get_settings() + +# Setup logging +setup_logging(settings.log_level) +logger = get_logger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """ + Application lifespan manager for startup/shutdown events + + Args: + app: FastAPI application instance + """ + # Startup + logger.info("=" * 60) + logger.info(f"Starting {settings.app_name} v{settings.app_version}") + logger.info(f"Debug mode: {settings.debug}") + logger.info(f"Log level: {settings.log_level}") + logger.info(f"Ollama URL: {settings.ollama_base_url}") + logger.info("=" * 60) + + # Check Ollama connectivity + ollama_client = get_ollama_client() + ollama_healthy = await ollama_client.health_check() + if ollama_healthy: + logger.info("✓ Ollama connection successful") + else: + logger.warning("✗ Ollama connection failed - AI features may not work") + + # Initialize security (OIDC authentication) + initialize_oidc(settings) + + yield + + # Shutdown + logger.info("Shutting down application") + await close_ollama_client() + + +# Create FastAPI application +app = FastAPI( + title=settings.app_name, + version=settings.app_version, + description=""" + Core Code API provides OpenAPI-compatible functions and AI orchestration for Open WebUI. + + ## Features + + ### OpenAI-Compatible API (v1) + - `/v1/chat/completions` - Chat completions with streaming support + - `/v1/models` - List available models + Compatible with OpenAI client libraries and Open WebUI. + + ### Conversation Memory (Phase 2) + - `/v1/conversations/{id}` - Get conversation history + - `/v1/conversations/{id}/search` - Semantic search within conversation + - `/v1/conversations/search` - Search across all conversations + - `/v1/conversations/{id}/stats` - Get conversation statistics + - `/v1/conversations/{id}/consolidate` - Manual consolidation + - `DELETE /v1/conversations/{id}` - Delete conversation + + Multi-tier memory system: + - **Tier 1**: Fast in-memory buffer (last 10 turns) + - **Tier 2/3**: Unified Qdrant storage (persistent + semantic search) + + ### Infrastructure Management + **Read Endpoints:** + - `GET /infrastructure/health` - Check Portainer & NPM connectivity + - `GET /infrastructure/services` - List all deployed services + - `GET /infrastructure/services/{name}` - Get service details + - `GET /infrastructure/ports` - List allocated ports + - `GET /infrastructure/domains` - List configured domains + + **Write Endpoints (Admin Only):** + - `POST /infrastructure/services` - Deploy new service from compose YAML + - `PUT /infrastructure/services/{name}` - Update existing service + - `DELETE /infrastructure/services/{name}` - Remove service and stack + - `POST /infrastructure/proxy` - Create proxy host with optional SSL + + Automates infrastructure operations via Portainer and Nginx Proxy Manager APIs. + + ### Web Scraper + Intelligent web scraping with main content extraction. + Perfect for extracting articles, documentation, and blog posts for LLM consumption. + + ## Authentication + + When OIDC authentication is enabled (oidc_enabled=true in config): + - Infrastructure write endpoints require authentication + - Use OAuth2/OIDC bearer token from Authentik + - Admin group membership required for infrastructure operations + + ## Integration + + This API is designed to integrate with: + - **Open WebUI**: Direct OpenAI API compatibility + - **Open WebUI Functions**: Import via OpenAPI spec + - **Open WebUI Pipelines**: Use as data source + - **LangChain**: Compatible with standard HTTP tools + + ## Documentation + + - **OpenAPI Spec**: `/openapi.json` + - **Swagger UI**: `/docs` + - **ReDoc**: `/redoc` + """, + docs_url="/docs", + redoc_url="/redoc", + openapi_url="/openapi.json", + lifespan=lifespan, + debug=settings.debug, + swagger_ui_init_oauth={ + "clientId": settings.oidc_audience, + "usePkceWithAuthorizationCodeGrant": True, + } if settings.oidc_enabled else None +) + +# Add CORS middleware +app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_credentials=settings.cors_credentials, + allow_methods=settings.cors_methods, + allow_headers=settings.cors_headers, +) + + +# Include controller routers +app.include_router(health_controller.router) # / and /health +app.include_router(tools_controller.router) # /web-scraper/scrape +app.include_router(infrastructure_controller.router) # /infrastructure/* +app.include_router(static_controller.router) # /static/* +app.include_router(ai_router) # /ai/* + + +# Global exception handler +@app.exception_handler(Exception) +async def global_exception_handler(request, exc): + """ + Catch-all exception handler for unhandled errors + + Args: + request: The request that caused the exception + exc: The exception instance + + Returns: + JSON error response + """ + logger.error(f"Unhandled exception: {str(exc)}", exc_info=True) + return JSONResponse( + status_code=500, + content={ + "detail": "Internal server error", + "type": type(exc).__name__ + } + ) diff --git a/src/models/__init__.py b/src/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/models/embeddings.py b/src/models/embeddings.py new file mode 100644 index 0000000..7e84061 --- /dev/null +++ b/src/models/embeddings.py @@ -0,0 +1,128 @@ +""" +Embedding model client for text vectorization + +Uses sentence-transformers for generating embeddings. +""" +import logging +from typing import List, Optional +from sentence_transformers import SentenceTransformer +from src.config import get_settings + +logger = logging.getLogger(__name__) +settings = get_settings() + + +class EmbeddingClient: + """Client for generating text embeddings""" + + def __init__(self, model_name: Optional[str] = None): + """ + Initialize embedding client + + Args: + model_name: Optional model name, defaults to config + """ + self.model_name = model_name or settings.embedding_model + self.dimension = settings.embedding_dimension + self._model: Optional[SentenceTransformer] = None + logger.info(f"Initializing EmbeddingClient with model: {self.model_name}") + + def _load_model(self) -> SentenceTransformer: + """ + Lazy load the embedding model + + Returns: + Loaded SentenceTransformer model + """ + if self._model is None: + logger.info(f"Loading embedding model: {self.model_name}") + self._model = SentenceTransformer(self.model_name) + logger.info(f"Model loaded successfully. Embedding dimension: {self.dimension}") + return self._model + + def embed_text(self, text: str) -> List[float]: + """ + Generate embedding for a single text + + Args: + text: Input text to embed + + Returns: + List of floats representing the embedding vector + """ + model = self._load_model() + embedding = model.encode(text, convert_to_numpy=True) + return embedding.tolist() + + def embed_batch(self, texts: List[str]) -> List[List[float]]: + """ + Generate embeddings for multiple texts + + Args: + texts: List of input texts + + Returns: + List of embedding vectors + """ + model = self._load_model() + embeddings = model.encode( + texts, + batch_size=settings.embedding_batch_size, + convert_to_numpy=True, + show_progress_bar=False + ) + return embeddings.tolist() + + def get_dimension(self) -> int: + """ + Get embedding dimension + + Returns: + Embedding vector dimension + """ + return self.dimension + + +# Global instance +_embedding_client: Optional[EmbeddingClient] = None + + +def get_embedding_client() -> EmbeddingClient: + """ + Get or create global embedding client instance + + Returns: + EmbeddingClient instance + """ + global _embedding_client + if _embedding_client is None: + _embedding_client = EmbeddingClient() + return _embedding_client + + +async def embed_text_async(text: str) -> List[float]: + """ + Async wrapper for embedding text + + Args: + text: Input text + + Returns: + Embedding vector + """ + client = get_embedding_client() + return client.embed_text(text) + + +async def embed_batch_async(texts: List[str]) -> List[List[float]]: + """ + Async wrapper for batch embedding + + Args: + texts: List of input texts + + Returns: + List of embedding vectors + """ + client = get_embedding_client() + return client.embed_batch(texts) diff --git a/src/models/embeddings_ollama.py b/src/models/embeddings_ollama.py new file mode 100644 index 0000000..eca12e6 --- /dev/null +++ b/src/models/embeddings_ollama.py @@ -0,0 +1,136 @@ +""" +Ollama-based embedding client for text vectorization + +Uses Ollama's embedding API instead of local sentence-transformers. +This eliminates the need for PyTorch and heavy ML dependencies. +""" +import logging +import httpx +from typing import List, Optional +from src.config import get_settings + +logger = logging.getLogger(__name__) +settings = get_settings() + + +class OllamaEmbeddingClient: + """Client for generating text embeddings using Ollama""" + + def __init__( + self, + model_name: Optional[str] = None, + base_url: Optional[str] = None, + timeout: int = 30 + ): + """ + Initialize Ollama embedding client + + Args: + model_name: Embedding model name (default: nomic-embed-text) + base_url: Ollama base URL (default from settings) + timeout: Request timeout in seconds + """ + self.model_name = model_name or settings.embedding_model + self.base_url = (base_url or settings.ollama_base_url).rstrip("/") + self.timeout = timeout + self.dimension = settings.embedding_dimension + + logger.info(f"Initializing OllamaEmbeddingClient with model: {self.model_name}") + logger.info(f"Ollama URL: {self.base_url}") + + async def embed_text(self, text: str) -> List[float]: + """ + Generate embedding for a single text using Ollama + + Args: + text: Input text to embed + + Returns: + List of floats representing the embedding vector + """ + try: + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + f"{self.base_url}/api/embeddings", + json={ + "model": self.model_name, + "prompt": text + } + ) + response.raise_for_status() + result = response.json() + return result["embedding"] + + except Exception as e: + logger.error(f"Error generating embedding via Ollama: {e}") + raise + + async def embed_batch(self, texts: List[str]) -> List[List[float]]: + """ + Generate embeddings for multiple texts + + Args: + texts: List of input texts + + Returns: + List of embedding vectors + """ + embeddings = [] + for text in texts: + embedding = await self.embed_text(text) + embeddings.append(embedding) + return embeddings + + def get_dimension(self) -> int: + """ + Get embedding dimension + + Returns: + Embedding vector dimension + """ + return self.dimension + + +# Global instance +_embedding_client: Optional[OllamaEmbeddingClient] = None + + +def get_embedding_client() -> OllamaEmbeddingClient: + """ + Get or create global Ollama embedding client instance + + Returns: + OllamaEmbeddingClient instance + """ + global _embedding_client + if _embedding_client is None: + _embedding_client = OllamaEmbeddingClient() + return _embedding_client + + +async def embed_text_async(text: str) -> List[float]: + """ + Async wrapper for embedding text + + Args: + text: Input text + + Returns: + Embedding vector + """ + client = get_embedding_client() + return await client.embed_text(text) + + +async def embed_batch_async(texts: List[str]) -> List[List[float]]: + """ + Async wrapper for batch embedding + + Args: + texts: List of input texts + + Returns: + List of embedding vectors + """ + client = get_embedding_client() + return await client.embed_batch(texts) diff --git a/src/models/ollama_client.py b/src/models/ollama_client.py new file mode 100644 index 0000000..5a07199 --- /dev/null +++ b/src/models/ollama_client.py @@ -0,0 +1,223 @@ +""" +Ollama client for model inference. +Handles both streaming and non-streaming requests. +""" + +import httpx +import json +import logging +from typing import AsyncIterator, Dict, Any, Optional +from src.config import get_settings + +logger = logging.getLogger(__name__) +settings = get_settings() + + +class OllamaClient: + """Client for interacting with Ollama API.""" + + def __init__(self): + self.base_url = settings.ollama_base_url + self.timeout = settings.ollama_timeout + self.client = httpx.AsyncClient(timeout=self.timeout) + logger.info(f"Initialized Ollama client: {self.base_url}") + + async def close(self): + """Close the HTTP client.""" + await self.client.aclose() + + def resolve_model(self, model_name: str) -> str: + """ + Resolve model alias to actual Ollama model. + + Args: + model_name: Requested model name (e.g., "gpt-3.5-turbo") + + Returns: + Actual Ollama model name (e.g., "gemma:7b") + """ + resolved = settings.model_aliases.get(model_name, model_name) + if resolved != model_name: + logger.info(f"Model resolution: {model_name} → {resolved}") + return resolved + + async def generate_non_streaming( + self, + model: str, + prompt: str, + temperature: float = 0.7, + max_tokens: Optional[int] = None + ) -> Dict[str, Any]: + """ + Generate non-streaming response from Ollama using chat endpoint. + + Args: + model: Model name + prompt: User prompt + temperature: Sampling temperature + max_tokens: Maximum tokens to generate + + Returns: + Dict with 'response' and 'tokens' keys + """ + actual_model = self.resolve_model(model) + + payload = { + "model": actual_model, + "messages": [ + {"role": "user", "content": prompt} + ], + "stream": False, + "options": { + "temperature": temperature, + } + } + + if max_tokens: + payload["options"]["num_predict"] = max_tokens + + logger.debug(f"Ollama request to {actual_model}") + + try: + response = await self.client.post( + f"{self.base_url}/api/chat", + json=payload + ) + response.raise_for_status() + result = response.json() + + return { + "response": result.get("message", {}).get("content", ""), + "tokens": { + "prompt": result.get("prompt_eval_count", 0), + "completion": result.get("eval_count", 0), + "total": result.get("prompt_eval_count", 0) + result.get("eval_count", 0) + } + } + + except httpx.HTTPError as e: + logger.error(f"Ollama request failed: {e}") + raise + + async def generate_streaming( + self, + model: str, + prompt: str, + temperature: float = 0.7, + max_tokens: Optional[int] = None + ) -> AsyncIterator[str]: + """ + Generate streaming response from Ollama using chat endpoint. + + Args: + model: Model name + prompt: User prompt + temperature: Sampling temperature + max_tokens: Maximum tokens to generate + + Yields: + Token strings + """ + actual_model = self.resolve_model(model) + + payload = { + "model": actual_model, + "messages": [ + {"role": "user", "content": prompt} + ], + "stream": True, + "options": { + "temperature": temperature, + } + } + + if max_tokens: + payload["options"]["num_predict"] = max_tokens + + logger.debug(f"Ollama streaming request to {actual_model}") + + try: + async with self.client.stream( + "POST", + f"{self.base_url}/api/chat", + json=payload + ) as response: + response.raise_for_status() + + async for line in response.aiter_lines(): + if not line: + continue + + try: + chunk = json.loads(line) + if "message" in chunk: + content = chunk["message"].get("content", "") + if content: + yield content + + # Check if done + if chunk.get("done", False): + break + + except json.JSONDecodeError: + logger.warning(f"Failed to parse JSON: {line}") + continue + + except httpx.HTTPError as e: + logger.error(f"Ollama streaming request failed: {e}") + raise + + async def health_check(self) -> bool: + """ + Check if Ollama is healthy. + + Returns: + True if healthy, False otherwise + """ + try: + response = await self.client.get( + f"{self.base_url}/api/tags", + timeout=5.0 + ) + return response.status_code == 200 + except Exception as e: + logger.error(f"Ollama health check failed: {e}") + return False + + async def list_models(self) -> Dict[str, Any]: + """ + List all available models in Ollama. + + Returns: + Dict with 'models' key containing list of model info + """ + try: + response = await self.client.get( + f"{self.base_url}/api/tags", + timeout=5.0 + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.error(f"Failed to list Ollama models: {e}") + raise + + +# Global client instance +_ollama_client: Optional[OllamaClient] = None + + +def get_ollama_client() -> OllamaClient: + """Get or create the global Ollama client instance.""" + global _ollama_client + if _ollama_client is None: + _ollama_client = OllamaClient() + return _ollama_client + + +async def close_ollama_client(): + """Close the global Ollama client.""" + global _ollama_client + if _ollama_client is not None: + await _ollama_client.close() + _ollama_client = None diff --git a/src/security.py b/src/security.py new file mode 100644 index 0000000..3f2d322 --- /dev/null +++ b/src/security.py @@ -0,0 +1,32 @@ +""" +Security initialization module + +Handles OIDC configuration and authentication setup +""" +from src.config import Settings +from src.auth.oidc import oidc_config +from src.logging_config import get_logger + +logger = get_logger(__name__) + + +def initialize_oidc(settings: Settings) -> None: + """ + Initialize OIDC authentication configuration + + Configures the global oidc_config instance with settings from environment. + If OIDC is enabled, logs the issuer URL for verification. + + Args: + settings: Application settings containing OIDC configuration + """ + oidc_config.configure( + enabled=settings.oidc_enabled, + issuer=settings.oidc_issuer, + audience=settings.oidc_audience + ) + + if settings.oidc_enabled: + logger.info(f"✓ OIDC authentication enabled (issuer: {settings.oidc_issuer})") + else: + logger.info("○ OIDC authentication disabled - API is publicly accessible") diff --git a/src/service_groups.py b/src/service_groups.py new file mode 100644 index 0000000..55512a4 --- /dev/null +++ b/src/service_groups.py @@ -0,0 +1,136 @@ +""" +Service Groups and Safety Configuration + +Defines service groups, dependencies, and always-on infrastructure services. +""" +from typing import List, Dict, Set + +# Always-on infrastructure services (CANNOT be stopped via API) +ALWAYS_ON_SERVICES: Set[str] = { + "portainer", + "nginx-proxy-manager", + "core-api", + "uptime-kuma", + "organizr", + "headscale", + "watchtower", + "netdata", + "maintenance", + "postgres-shared", + "redis-shared", + "authentik", +} + +# Service groups - services that should be started/stopped together +SERVICE_GROUPS: Dict[str, List[str]] = { + "jellyfin": [ + "jellyfin", + ], + "nextcloud": [ + "nextcloud", + ], + "gitea": [ + "gitea", + "gitea-db", + ], + "ai-stack": [ + "open-webui", + "ollama", + "qdrant", + ], + "samba": [ + "samba", + ], +} + +# Reverse mapping: service name -> group name +SERVICE_TO_GROUP: Dict[str, str] = {} +for group, services in SERVICE_GROUPS.items(): + for service in services: + SERVICE_TO_GROUP[service] = group + + +def is_always_on(service_name: str) -> bool: + """ + Check if a service is marked as always-on (infrastructure) + + Args: + service_name: Name of the service + + Returns: + True if service cannot be stopped, False otherwise + """ + return service_name.lower() in ALWAYS_ON_SERVICES + + +def get_service_group(service_name: str) -> List[str]: + """ + Get all services in the same group as the given service + + Args: + service_name: Name of the service + + Returns: + List of service names in the group (including the service itself) + Returns [service_name] if not part of a group + """ + group = SERVICE_TO_GROUP.get(service_name.lower()) + if group: + return SERVICE_GROUPS[group].copy() + return [service_name] + + +def get_group_name(service_name: str) -> str: + """ + Get the group name for a service + + Args: + service_name: Name of the service + + Returns: + Group name or the service name if not in a group + """ + return SERVICE_TO_GROUP.get(service_name.lower(), service_name) + + +def list_service_groups() -> Dict[str, List[str]]: + """ + Get all defined service groups + + Returns: + Dictionary of group names to service lists + """ + return SERVICE_GROUPS.copy() + + +def list_stoppable_services() -> List[str]: + """ + Get list of all services that can be stopped + + Returns: + List of service names that are not always-on + """ + stoppable = [] + for services in SERVICE_GROUPS.values(): + stoppable.extend(services) + + # Remove any always-on services (shouldn't be in groups, but safety check) + return [s for s in stoppable if not is_always_on(s)] + + +def validate_stop_request(service_names: List[str]) -> tuple[bool, str]: + """ + Validate that a list of services can be stopped + + Args: + service_names: List of service names to check + + Returns: + Tuple of (is_valid, error_message) + error_message is empty string if valid + """ + for service in service_names: + if is_always_on(service): + return False, f"Cannot stop always-on service: {service}" + + return True, "" diff --git a/src/web_scraper/__init__.py b/src/web_scraper/__init__.py new file mode 100644 index 0000000..fa69ae4 --- /dev/null +++ b/src/web_scraper/__init__.py @@ -0,0 +1,13 @@ +""" +Web scraper module for extracting content from websites +""" +from src.web_scraper.router import router +from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse +from src.web_scraper.service import WebScraperService + +__all__ = [ + "router", + "WebScraperRequest", + "WebScraperResponse", + "WebScraperService", +] diff --git a/src/web_scraper/config.py b/src/web_scraper/config.py new file mode 100644 index 0000000..70c896c --- /dev/null +++ b/src/web_scraper/config.py @@ -0,0 +1,32 @@ +""" +Configuration for web scraper module +""" +from pydantic_settings import BaseSettings +from functools import lru_cache + + +class WebScraperSettings(BaseSettings): + """Web scraper specific settings""" + + # HTTP client configuration + request_timeout: int = 30 + max_redirects: int = 5 + user_agent: str = "Mozilla/5.0 (compatible; CoreCode/1.0)" + + # Content extraction + default_max_length: int = 10000 + max_links_to_extract: int = 50 + + # Rate limiting (future use) + rate_limit_enabled: bool = False + requests_per_minute: int = 60 + + class Config: + env_prefix = "WEB_SCRAPER_" + case_sensitive = False + + +@lru_cache() +def get_web_scraper_settings() -> WebScraperSettings: + """Cached web scraper settings instance""" + return WebScraperSettings() diff --git a/src/web_scraper/exceptions.py b/src/web_scraper/exceptions.py new file mode 100644 index 0000000..3fb5443 --- /dev/null +++ b/src/web_scraper/exceptions.py @@ -0,0 +1,18 @@ +""" +Custom exceptions for web scraper module +""" + + +class WebScraperException(Exception): + """Base exception for web scraper module""" + pass + + +class FetchError(WebScraperException): + """Raised when URL fetch fails""" + pass + + +class ScrapingError(WebScraperException): + """Raised when content extraction fails""" + pass diff --git a/src/web_scraper/router.py b/src/web_scraper/router.py new file mode 100644 index 0000000..afb3065 --- /dev/null +++ b/src/web_scraper/router.py @@ -0,0 +1,79 @@ +""" +API routes for web scraper module +""" +from fastapi import APIRouter, HTTPException, status + +from src.logging_config import get_logger +from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse +from src.web_scraper.service import WebScraperService +from src.web_scraper.exceptions import FetchError, ScrapingError + +logger = get_logger(__name__) + +router = APIRouter( + prefix="/web-scraper", + tags=["Web Scraper"] +) + +# Initialize service (could be dependency injected for testing) +scraper_service = WebScraperService() + + +@router.post( + "/scrape", + response_model=WebScraperResponse, + status_code=status.HTTP_200_OK, + summary="Scrape website content", + description=""" + Scrape and extract main content from a website. + + Uses trafilatura for intelligent content extraction (articles, blog posts, documentation), + with BeautifulSoup as fallback. Perfect for feeding webpage content to LLMs. + + **Features:** + - Intelligent main content extraction + - Removes navigation, ads, footers + - Optional link extraction + - Configurable content length limits + + **Rate Limiting:** None (internal network use only) + """ +) +async def scrape_website(request: WebScraperRequest) -> WebScraperResponse: + """ + Scrape a website and extract its main content + + Args: + request: Scraping request with URL and options + + Returns: + Extracted content with metadata + + Raises: + HTTPException: 400 for fetch errors, 500 for processing errors + """ + try: + logger.info(f"Received scrape request for: {request.url}") + result = await scraper_service.scrape_url(request) + return result + + except FetchError as e: + logger.warning(f"Fetch failed: {str(e)}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Failed to fetch URL: {str(e)}" + ) + + except ScrapingError as e: + logger.error(f"Scraping failed: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to extract content: {str(e)}" + ) + + except Exception as e: + logger.error(f"Unexpected error: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="An unexpected error occurred" + ) diff --git a/src/web_scraper/schemas.py b/src/web_scraper/schemas.py new file mode 100644 index 0000000..9afde82 --- /dev/null +++ b/src/web_scraper/schemas.py @@ -0,0 +1,69 @@ +""" +Pydantic schemas for web scraper module +""" +from pydantic import HttpUrl, Field +from typing import Optional +from datetime import datetime +from src.base_schema import BaseSchema + + +class WebScraperRequest(BaseSchema): + """Request model for web scraping""" + + url: HttpUrl = Field( + ..., + description="The URL to scrape", + examples=["https://example.com/article"] + ) + + extract_main_content: bool = Field( + default=True, + description="Use intelligent content extraction (trafilatura) vs raw HTML parsing" + ) + + include_links: bool = Field( + default=False, + description="Include list of links found on the page" + ) + + max_length: Optional[int] = Field( + default=10000, + ge=100, + le=100000, + description="Maximum content length to return (100-100000 chars)" + ) + + +class WebScraperResponse(BaseSchema): + """Response model for web scraping""" + + url: str = Field( + ..., + description="The scraped URL" + ) + + title: Optional[str] = Field( + default=None, + description="Page title extracted from tag" + ) + + content: str = Field( + ..., + description="Extracted page content" + ) + + extracted_at: datetime = Field( + ..., + description="UTC timestamp when content was extracted" + ) + + content_length: int = Field( + ..., + ge=0, + description="Length of extracted content in characters" + ) + + links: Optional[list[str]] = Field( + default=None, + description="List of HTTP(S) links found on the page (max 50)" + ) diff --git a/src/web_scraper/service.py b/src/web_scraper/service.py new file mode 100644 index 0000000..f45468a --- /dev/null +++ b/src/web_scraper/service.py @@ -0,0 +1,213 @@ +""" +Business logic for web scraper module +""" +import httpx +from bs4 import BeautifulSoup +import trafilatura +from datetime import datetime, timezone +from typing import Optional + +from src.logging_config import get_logger +from src.web_scraper.config import get_web_scraper_settings +from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse +from src.web_scraper.exceptions import ScrapingError, FetchError + +logger = get_logger(__name__) + + +class WebScraperService: + """Service class for web scraping operations""" + + def __init__(self): + self.settings = get_web_scraper_settings() + + async def scrape_url(self, request: WebScraperRequest) -> WebScraperResponse: + """ + Scrape and extract content from a URL + + Args: + request: Scraping request parameters + + Returns: + Extracted content with metadata + + Raises: + FetchError: If URL cannot be fetched + ScrapingError: If content extraction fails + """ + url_str = str(request.url) + logger.info(f"Starting scrape for URL: {url_str}") + + try: + # Fetch the webpage + html_content = await self._fetch_url(url_str) + + # Extract content based on settings + if request.extract_main_content: + content = self._extract_main_content(html_content, request.include_links) + else: + content = self._extract_basic_content(html_content) + + # Extract metadata + title = self._extract_title(html_content) + links = self._extract_links(html_content) if request.include_links else None + + # Clean and truncate content + content = self._clean_content(content) + if request.max_length and len(content) > request.max_length: + content = content[:request.max_length] + "\n\n[Content truncated...]" + logger.debug(f"Content truncated to {request.max_length} characters") + + logger.info(f"Successfully scraped {len(content)} characters from {url_str}") + + return WebScraperResponse( + url=url_str, + title=title, + content=content, + extracted_at=datetime.now(timezone.utc), + content_length=len(content), + links=links + ) + + except FetchError: + raise + except Exception as e: + logger.error(f"Scraping failed for {url_str}: {str(e)}", exc_info=True) + raise ScrapingError(f"Failed to scrape content: {str(e)}") + + async def _fetch_url(self, url: str) -> str: + """ + Fetch HTML content from URL + + Args: + url: URL to fetch + + Returns: + HTML content as string + + Raises: + FetchError: If fetch fails + """ + try: + async with httpx.AsyncClient( + timeout=self.settings.request_timeout, + follow_redirects=True, + max_redirects=self.settings.max_redirects + ) as client: + logger.debug(f"Fetching URL: {url}") + response = await client.get( + url, + headers={"User-Agent": self.settings.user_agent} + ) + response.raise_for_status() + logger.debug(f"Fetched {len(response.text)} bytes from {url}") + return response.text + + except httpx.HTTPStatusError as e: + logger.error(f"HTTP error {e.response.status_code} for {url}") + raise FetchError(f"HTTP {e.response.status_code}: {e.response.reason_phrase}") + except httpx.RequestError as e: + logger.error(f"Request error for {url}: {str(e)}") + raise FetchError(f"Failed to fetch URL: {str(e)}") + + def _extract_main_content(self, html: str, include_links: bool = False) -> str: + """ + Extract main content using trafilatura (intelligent extraction) + + Args: + html: Raw HTML content + include_links: Whether to preserve links in output + + Returns: + Extracted content + """ + logger.debug("Extracting main content with trafilatura") + content = trafilatura.extract( + html, + include_links=include_links, + include_images=False, + output_format='txt', + no_fallback=False + ) + + # Fallback to BeautifulSoup if trafilatura fails + if not content: + logger.debug("Trafilatura extraction failed, falling back to BeautifulSoup") + content = self._extract_basic_content(html) + + return content + + def _extract_basic_content(self, html: str) -> str: + """ + Extract content using basic BeautifulSoup parsing + + Args: + html: Raw HTML content + + Returns: + Extracted text content + """ + logger.debug("Extracting content with BeautifulSoup") + soup = BeautifulSoup(html, 'html.parser') + + # Remove unwanted elements + for element in soup(["script", "style", "nav", "footer", "header", "aside"]): + element.decompose() + + # Extract text + text = soup.get_text(separator='\n', strip=True) + return text + + def _extract_title(self, html: str) -> Optional[str]: + """ + Extract page title from HTML + + Args: + html: Raw HTML content + + Returns: + Page title or None + """ + soup = BeautifulSoup(html, 'html.parser') + title = soup.title.string if soup.title else None + if title: + title = title.strip() + logger.debug(f"Extracted title: {title}") + return title + + def _extract_links(self, html: str) -> list[str]: + """ + Extract HTTP(S) links from HTML + + Args: + html: Raw HTML content + + Returns: + List of absolute HTTP(S) URLs + """ + soup = BeautifulSoup(html, 'html.parser') + links = [ + a.get('href') + for a in soup.find_all('a', href=True) + if a.get('href', '').startswith('http') + ] + + # Limit number of links + links = links[:self.settings.max_links_to_extract] + logger.debug(f"Extracted {len(links)} links") + return links + + def _clean_content(self, content: str) -> str: + """ + Clean and normalize extracted content + + Args: + content: Raw extracted content + + Returns: + Cleaned content + """ + # Remove empty lines and normalize whitespace + lines = [line.strip() for line in content.split('\n') if line.strip()] + cleaned = '\n'.join(lines) + return cleaned diff --git a/static/widgets/ai-stats.html b/static/widgets/ai-stats.html new file mode 100644 index 0000000..6ab68c3 --- /dev/null +++ b/static/widgets/ai-stats.html @@ -0,0 +1,491 @@ +<!DOCTYPE html> +<html> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>AI Performance Stats + + + +
+
+
+ Loading AI performance metrics... +
+
+ + + + diff --git a/static/widgets/service-control.html b/static/widgets/service-control.html new file mode 100644 index 0000000..ec2a350 --- /dev/null +++ b/static/widgets/service-control.html @@ -0,0 +1,487 @@ + + + + + + Service Control + + + +
+
+ +
+
🎛️ Stoppable Services
+
Loading services...
+
+ +
+
🔒 Always-On Infrastructure
+
Loading infrastructure...
+
+
+ + + + diff --git a/tests/test_memory_integration.py b/tests/test_memory_integration.py new file mode 100644 index 0000000..c309a1f --- /dev/null +++ b/tests/test_memory_integration.py @@ -0,0 +1,269 @@ +""" +Integration tests for Phase 2 Memory System + +Tests the complete memory stack: +- Tier 1: ConversationBufferMemory +- Tier 2/3: QdrantConversationMemory +- Embedding Client +""" +import asyncio +import pytest +from datetime import datetime +from src.memory import ( + ConversationBufferMemory, + QdrantConversationMemory, + ConversationTurn, + MessageRole, + TokenUsage, + get_buffer_memory, + get_qdrant_memory +) +from src.models.embeddings import get_embedding_client + + +class TestEmbeddingClient: + """Test embedding generation""" + + def test_embedding_client_init(self): + """Test embedding client initialization""" + client = get_embedding_client() + assert client is not None + assert client.dimension == 384 + print(f"✓ Embedding client initialized: {client.model_name}") + + def test_single_embedding(self): + """Test single text embedding""" + client = get_embedding_client() + text = "Hello, this is a test message for embedding generation" + + embedding = client.embed_text(text) + + assert isinstance(embedding, list) + assert len(embedding) == 384 + assert all(isinstance(x, float) for x in embedding) + print(f"✓ Single embedding generated: {len(embedding)} dimensions") + + def test_batch_embedding(self): + """Test batch text embedding""" + client = get_embedding_client() + texts = [ + "First message about Python programming", + "Second message about machine learning", + "Third message about data science" + ] + + embeddings = client.embed_batch(texts) + + assert len(embeddings) == 3 + assert all(len(emb) == 384 for emb in embeddings) + print(f"✓ Batch embeddings generated: {len(embeddings)} texts") + + +class TestQdrantMemory: + """Test Qdrant memory storage and retrieval""" + + @pytest.fixture + def qdrant_memory(self): + """Get Qdrant memory instance""" + return get_qdrant_memory() + + @pytest.fixture + def test_conversation_id(self): + """Generate unique test conversation ID""" + return f"test_conv_{int(datetime.utcnow().timestamp())}" + + @pytest.mark.asyncio + async def test_qdrant_connection(self, qdrant_memory): + """Test Qdrant connection and collection""" + assert qdrant_memory.client is not None + assert qdrant_memory.collection_name == "core_api_conversations" + print(f"✓ Connected to Qdrant: {qdrant_memory.host}:{qdrant_memory.port}") + + @pytest.mark.asyncio + async def test_add_turn(self, qdrant_memory, test_conversation_id): + """Test adding a turn to Qdrant""" + turn = ConversationTurn( + role=MessageRole.USER, + content="What is Python?", + turn_number=1, + tokens=TokenUsage(prompt=10, completion=0, total=10) + ) + + await qdrant_memory.add_turn(test_conversation_id, turn) + + # Verify it was stored + exists = await qdrant_memory.conversation_exists(test_conversation_id) + assert exists is True + print(f"✓ Turn stored in Qdrant: {test_conversation_id}") + + @pytest.mark.asyncio + async def test_chronological_retrieval(self, qdrant_memory, test_conversation_id): + """Test Tier 2 mode: chronological retrieval""" + # Add multiple turns + turns = [ + ConversationTurn(role=MessageRole.USER, content="What is Python?", turn_number=1), + ConversationTurn(role=MessageRole.ASSISTANT, content="Python is a programming language", turn_number=2), + ConversationTurn(role=MessageRole.USER, content="How do I learn it?", turn_number=3), + ] + + for turn in turns: + await qdrant_memory.add_turn(test_conversation_id, turn) + + # Retrieve turns chronologically + retrieved = await qdrant_memory.get_turns(test_conversation_id) + + assert len(retrieved) == 3 + assert retrieved[0].turn_number == 1 + assert retrieved[1].turn_number == 2 + assert retrieved[2].turn_number == 3 + assert retrieved[0].content == "What is Python?" + print(f"✓ Chronological retrieval works: {len(retrieved)} turns") + + @pytest.mark.asyncio + async def test_semantic_search(self, qdrant_memory, test_conversation_id): + """Test Tier 3 mode: semantic search""" + # Add turns with distinct topics + turns = [ + ConversationTurn(role=MessageRole.USER, content="I love machine learning and neural networks", turn_number=10), + ConversationTurn(role=MessageRole.USER, content="Pizza is my favorite food", turn_number=11), + ConversationTurn(role=MessageRole.USER, content="Deep learning models are fascinating", turn_number=12), + ] + + for turn in turns: + await qdrant_memory.add_turn(test_conversation_id, turn) + + # Search for AI-related content + results = await qdrant_memory.similarity_search( + query="artificial intelligence and AI", + conversation_id=test_conversation_id, + limit=3 + ) + + assert len(results) > 0 + # Top results should be about ML/AI, not pizza + top_result = results[0] + assert "machine learning" in top_result["content"] or "Deep learning" in top_result["content"] + assert top_result["score"] > 0.5 # Reasonable similarity score + print(f"✓ Semantic search works: {len(results)} matches, top score: {results[0]['score']:.3f}") + + @pytest.mark.asyncio + async def test_conversation_stats(self, qdrant_memory, test_conversation_id): + """Test conversation statistics""" + stats = await qdrant_memory.get_conversation_stats(test_conversation_id) + + assert stats["conversation_id"] == test_conversation_id + assert stats["total_turns"] >= 0 + assert "total_tokens" in stats + print(f"✓ Stats retrieved: {stats['total_turns']} turns, {stats['total_tokens']} tokens") + + @pytest.mark.asyncio + async def test_clear_conversation(self, qdrant_memory, test_conversation_id): + """Test clearing a conversation""" + # Add a turn + turn = ConversationTurn(role=MessageRole.USER, content="Test message", turn_number=99) + await qdrant_memory.add_turn(test_conversation_id, turn) + + # Clear it + await qdrant_memory.clear_conversation(test_conversation_id) + + # Verify it's gone + exists = await qdrant_memory.conversation_exists(test_conversation_id) + assert exists is False + print(f"✓ Conversation cleared: {test_conversation_id}") + + +class TestIntegration: + """Test full integration: Tier 1 + Qdrant + Embeddings""" + + @pytest.mark.asyncio + async def test_full_memory_flow(self): + """Test complete memory flow: Buffer → Qdrant""" + conversation_id = f"integration_test_{int(datetime.utcnow().timestamp())}" + + # Initialize both tiers + buffer_memory = get_buffer_memory() + qdrant_memory = get_qdrant_memory() + + # 1. Add turns to buffer (Tier 1) + turns = [ + ConversationTurn(role=MessageRole.USER, content="Hello!", turn_number=1), + ConversationTurn(role=MessageRole.ASSISTANT, content="Hi there!", turn_number=2), + ConversationTurn(role=MessageRole.USER, content="How are you?", turn_number=3), + ] + + for turn in turns: + await buffer_memory.add_turn(conversation_id, turn) + + # Verify buffer has them + buffer = await buffer_memory.get_buffer(conversation_id) + assert len(buffer.turns) == 3 + print(f"✓ Tier 1 buffer: {len(buffer.turns)} turns") + + # 2. Move to Qdrant (Tier 2/3) + for turn in buffer.turns: + await qdrant_memory.add_turn(conversation_id, turn) + + # Verify Qdrant has them + qdrant_turns = await qdrant_memory.get_turns(conversation_id) + assert len(qdrant_turns) == 3 + print(f"✓ Tier 2/3 Qdrant: {len(qdrant_turns)} turns") + + # 3. Test semantic search across both + search_results = await qdrant_memory.similarity_search( + query="greeting", + conversation_id=conversation_id, + limit=2 + ) + assert len(search_results) > 0 + print(f"✓ Semantic search: {len(search_results)} matches") + + # Cleanup + await qdrant_memory.clear_conversation(conversation_id) + await buffer_memory.clear_conversation(conversation_id) + print(f"✓ Full memory flow complete!") + + +def run_tests(): + """Run all tests""" + print("\n" + "="*60) + print("Phase 2 Memory System Integration Tests") + print("="*60 + "\n") + + # Test 1: Embedding Client + print("Test 1: Embedding Client") + print("-" * 40) + test_embed = TestEmbeddingClient() + test_embed.test_embedding_client_init() + test_embed.test_single_embedding() + test_embed.test_batch_embedding() + print() + + # Test 2: Qdrant Memory + print("Test 2: Qdrant Memory Storage") + print("-" * 40) + test_qdrant = TestQdrantMemory() + qdrant_memory = get_qdrant_memory() + test_conv_id = f"test_conv_{int(datetime.utcnow().timestamp())}" + + asyncio.run(test_qdrant.test_qdrant_connection(qdrant_memory)) + asyncio.run(test_qdrant.test_add_turn(qdrant_memory, test_conv_id)) + asyncio.run(test_qdrant.test_chronological_retrieval(qdrant_memory, test_conv_id)) + asyncio.run(test_qdrant.test_semantic_search(qdrant_memory, test_conv_id)) + asyncio.run(test_qdrant.test_conversation_stats(qdrant_memory, test_conv_id)) + asyncio.run(test_qdrant.test_clear_conversation(qdrant_memory, test_conv_id)) + print() + + # Test 3: Full Integration + print("Test 3: Full Integration (Tier 1 + Tier 2/3)") + print("-" * 40) + test_integration = TestIntegration() + asyncio.run(test_integration.test_full_memory_flow()) + print() + + print("="*60) + print("✅ All Memory System Tests Passed!") + print("="*60) + + +if __name__ == "__main__": + run_tests() diff --git a/tests/test_memory_manager.py b/tests/test_memory_manager.py new file mode 100644 index 0000000..dae3f58 --- /dev/null +++ b/tests/test_memory_manager.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +""" +Test MemoryManager orchestration + +Verifies unified memory interface works correctly. +""" +import asyncio +import sys +from datetime import datetime + +sys.path.insert(0, '/app') + +from src.memory import MemoryManager, get_memory_manager, MessageRole, TokenUsage + + +async def test_memory_manager(): + """Test MemoryManager orchestration""" + print("\n" + "="*60) + print("MEMORY MANAGER TEST") + print("="*60) + + test_conv_id = f"manager_test_{int(datetime.utcnow().timestamp())}" + + try: + # Initialize manager + manager = get_memory_manager() + print(f"✓ MemoryManager initialized") + + # Test 1: Add turns through manager + print("\n1. Adding turns via MemoryManager...") + turn1 = await manager.add_turn( + conversation_id=test_conv_id, + role=MessageRole.USER, + content="Hello, how are you?", + tokens=TokenUsage(prompt=5, completion=0, total=5) + ) + assert turn1.turn_number == 1 + print(f" ✓ Turn 1 added: {turn1.content[:30]}...") + + turn2 = await manager.add_turn( + conversation_id=test_conv_id, + role=MessageRole.ASSISTANT, + content="I'm doing great! How can I help you today?", + tokens=TokenUsage(prompt=5, completion=10, total=15) + ) + assert turn2.turn_number == 2 + print(f" ✓ Turn 2 added: {turn2.content[:30]}...") + + # Test 2: Get recent turns (from buffer) + print("\n2. Getting recent turns from buffer...") + recent = await manager.get_recent_turns(test_conv_id, limit=10) + assert len(recent) == 2 + assert recent[0].turn_number == 1 + assert recent[1].turn_number == 2 + print(f" ✓ Retrieved {len(recent)} recent turns from buffer") + + # Test 3: Add more turns to trigger consolidation (threshold = 10) + print("\n3. Adding turns to trigger auto-consolidation...") + for i in range(3, 11): # Add turns 3-10 + await manager.add_turn( + conversation_id=test_conv_id, + role=MessageRole.USER if i % 2 == 1 else MessageRole.ASSISTANT, + content=f"Test message number {i}", + tokens=TokenUsage(prompt=5, completion=5, total=10) + ) + print(f" ✓ Added 8 more turns (total: 10)") + + # Check if consolidation happened (turn 10 should trigger it) + print("\n4. Verifying auto-consolidation...") + stats = await manager.get_conversation_stats(test_conv_id) + print(f" Buffer turns: {stats['buffer_turns']}") + print(f" Qdrant turns: {stats['qdrant_turns']}") + print(f" Exists in buffer: {stats['exists_in_buffer']}") + print(f" Exists in Qdrant: {stats['exists_in_qdrant']}") + + if stats['qdrant_turns'] > 0: + print(f" ✓ Auto-consolidation triggered! {stats['qdrant_turns']} turns in Qdrant") + else: + print(f" ⚠ No auto-consolidation yet (threshold may not be reached)") + + # Test 4: Manual consolidation + print("\n5. Testing manual consolidation...") + consolidated = await manager.consolidate(test_conv_id) + print(f" ✓ Manually consolidated {consolidated} turns") + + # Test 5: Get full history (buffer + Qdrant) + print("\n6. Getting full conversation history...") + full_history = await manager.get_full_history(test_conv_id) + print(f" ✓ Retrieved {len(full_history)} total turns") + assert len(full_history) == 10, f"Expected 10 turns, got {len(full_history)}" + print(f" ✓ Full history verified (10 turns)") + + # Test 6: Semantic search + print("\n7. Testing semantic search...") + search_results = await manager.search_conversations( + query="greeting hello", + conversation_id=test_conv_id, + limit=3 + ) + if len(search_results) > 0: + print(f" ✓ Semantic search found {len(search_results)} matches") + print(f" Top: '{search_results[0]['content'][:40]}...' (score: {search_results[0]['score']:.3f})") + else: + print(f" ⚠ No semantic search results (may need more data)") + + # Test 7: Clear conversation + print("\n8. Clearing conversation...") + await manager.clear_conversation(test_conv_id) + stats_after = await manager.get_conversation_stats(test_conv_id) + assert stats_after['buffer_turns'] == 0 + assert stats_after['qdrant_turns'] == 0 + print(f" ✓ Conversation cleared from all tiers") + + print("\n" + "="*60) + print("✅ MEMORY MANAGER TEST: PASSED") + print("="*60) + print("\nMemoryManager verified:") + print(" ✓ Add turns with auto turn numbering") + print(" ✓ Get recent turns from buffer") + print(" ✓ Auto-consolidation (when threshold reached)") + print(" ✓ Manual consolidation") + print(" ✓ Get full history (buffer + Qdrant)") + print(" ✓ Semantic search") + print(" ✓ Clear conversation") + print(" ✓ Conversation stats") + return True + + except Exception as e: + print(f"\n❌ MEMORY MANAGER TEST: FAILED") + print(f"Error: {e}") + import traceback + traceback.print_exc() + + # Cleanup on error + try: + await manager.clear_conversation(test_conv_id) + except: + pass + + return False + + +def main(): + """Run the test""" + success = asyncio.run(test_memory_manager()) + return 0 if success else 1 + + +if __name__ == "__main__": + exit(main()) diff --git a/tests/test_memory_simple.py b/tests/test_memory_simple.py new file mode 100644 index 0000000..62daa4d --- /dev/null +++ b/tests/test_memory_simple.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +""" +Simple integration tests for Phase 2 Memory System +No external dependencies beyond the memory system itself +""" +import asyncio +import sys +from datetime import datetime + +# Add src to path +sys.path.insert(0, '/app') + +from src.memory import ( + ConversationBufferMemory, + QdrantConversationMemory, + ConversationTurn, + MessageRole, + TokenUsage, + get_buffer_memory, + get_qdrant_memory +) +from src.models.embeddings import get_embedding_client + + +def test_embedding_client(): + """Test 1: Embedding Client""" + print("\n" + "="*60) + print("Test 1: Embedding Client") + print("="*60) + + try: + # Initialize + client = get_embedding_client() + assert client is not None + assert client.dimension == 384 + print(f"✓ Embedding client initialized: {client.model_name}") + print(f"✓ Embedding dimension: {client.dimension}") + + # Single embedding + text = "Hello, this is a test message for embedding generation" + embedding = client.embed_text(text) + assert isinstance(embedding, list) + assert len(embedding) == 384 + assert all(isinstance(x, float) for x in embedding) + print(f"✓ Single embedding generated: {len(embedding)} dimensions") + print(f" Sample values: [{embedding[0]:.4f}, {embedding[1]:.4f}, {embedding[2]:.4f}, ...]") + + # Batch embedding + texts = [ + "First message about Python programming", + "Second message about machine learning", + "Third message about data science" + ] + embeddings = client.embed_batch(texts) + assert len(embeddings) == 3 + assert all(len(emb) == 384 for emb in embeddings) + print(f"✓ Batch embeddings generated: {len(embeddings)} texts") + + print("\n✅ Embedding Client Tests: PASSED") + return True + + except Exception as e: + print(f"\n❌ Embedding Client Tests: FAILED") + print(f"Error: {e}") + import traceback + traceback.print_exc() + return False + + +async def test_qdrant_memory(): + """Test 2: Qdrant Memory Storage""" + print("\n" + "="*60) + print("Test 2: Qdrant Memory Storage") + print("="*60) + + test_conv_id = f"test_conv_{int(datetime.utcnow().timestamp())}" + + try: + # Initialize + qdrant_memory = get_qdrant_memory() + assert qdrant_memory.client is not None + assert qdrant_memory.collection_name == "core_api_conversations" + print(f"✓ Connected to Qdrant: {qdrant_memory.host}:{qdrant_memory.port}") + print(f"✓ Collection: {qdrant_memory.collection_name}") + + # Add single turn + turn1 = ConversationTurn( + role=MessageRole.USER, + content="What is Python?", + turn_number=1, + tokens=TokenUsage(prompt=10, completion=0, total=10) + ) + await qdrant_memory.add_turn(test_conv_id, turn1) + print(f"✓ Turn 1 stored in Qdrant") + + # Verify it exists + exists = await qdrant_memory.conversation_exists(test_conv_id) + assert exists is True + print(f"✓ Conversation exists: {test_conv_id}") + + # Add more turns for chronological test + turn2 = ConversationTurn( + role=MessageRole.ASSISTANT, + content="Python is a high-level programming language known for simplicity and readability", + turn_number=2 + ) + turn3 = ConversationTurn( + role=MessageRole.USER, + content="How do I learn Python programming?", + turn_number=3 + ) + + await qdrant_memory.add_turn(test_conv_id, turn2) + await qdrant_memory.add_turn(test_conv_id, turn3) + print(f"✓ Turns 2-3 stored in Qdrant") + + # Test chronological retrieval (Tier 2 mode) + retrieved = await qdrant_memory.get_turns(test_conv_id) + assert len(retrieved) == 3 + assert retrieved[0].turn_number == 1 + assert retrieved[1].turn_number == 2 + assert retrieved[2].turn_number == 3 + assert retrieved[0].content == "What is Python?" + print(f"✓ Chronological retrieval works: {len(retrieved)} turns") + for i, turn in enumerate(retrieved, 1): + print(f" Turn {turn.turn_number}: {turn.role.value} - {turn.content[:50]}...") + + # Add turns with distinct topics for semantic search + turn10 = ConversationTurn( + role=MessageRole.USER, + content="I love machine learning and neural networks and artificial intelligence", + turn_number=10 + ) + turn11 = ConversationTurn( + role=MessageRole.USER, + content="Pizza is my favorite food and I enjoy eating pasta", + turn_number=11 + ) + turn12 = ConversationTurn( + role=MessageRole.USER, + content="Deep learning models and transformers are fascinating AI technologies", + turn_number=12 + ) + + await qdrant_memory.add_turn(test_conv_id, turn10) + await qdrant_memory.add_turn(test_conv_id, turn11) + await qdrant_memory.add_turn(test_conv_id, turn12) + print(f"✓ Added 3 more turns for semantic search test") + + # Test semantic search (Tier 3 mode) + search_results = await qdrant_memory.similarity_search( + query="artificial intelligence and deep learning", + conversation_id=test_conv_id, + limit=3 + ) + assert len(search_results) > 0 + print(f"✓ Semantic search works: {len(search_results)} matches") + + # Top result should be about AI/ML, not food + top_result = search_results[0] + print(f" Top match (score: {top_result['score']:.3f}): {top_result['content'][:60]}...") + assert top_result["score"] > 0.5, "Semantic similarity score too low" + + # Verify top matches are AI-related + ai_keywords = ["machine learning", "neural networks", "Deep learning", "AI", "artificial intelligence"] + top_content = search_results[0]["content"] + assert any(keyword in top_content for keyword in ai_keywords), "Top result not AI-related" + print(f"✓ Semantic relevance verified (AI-related content ranked higher)") + + # Test conversation stats + stats = await qdrant_memory.get_conversation_stats(test_conv_id) + assert stats["conversation_id"] == test_conv_id + assert stats["total_turns"] == 6 + print(f"✓ Stats retrieved: {stats['total_turns']} turns, {stats['total_tokens']} tokens") + + # Cleanup + await qdrant_memory.clear_conversation(test_conv_id) + exists_after = await qdrant_memory.conversation_exists(test_conv_id) + assert exists_after is False + print(f"✓ Conversation cleared successfully") + + print("\n✅ Qdrant Memory Tests: PASSED") + return True + + except Exception as e: + print(f"\n❌ Qdrant Memory Tests: FAILED") + print(f"Error: {e}") + import traceback + traceback.print_exc() + + # Cleanup on error + try: + await qdrant_memory.clear_conversation(test_conv_id) + except: + pass + + return False + + +async def test_full_integration(): + """Test 3: Full Integration (Tier 1 + Tier 2/3)""" + print("\n" + "="*60) + print("Test 3: Full Integration (Tier 1 + Tier 2/3)") + print("="*60) + + test_conv_id = f"integration_test_{int(datetime.utcnow().timestamp())}" + + try: + # Initialize both tiers + buffer_memory = get_buffer_memory() + qdrant_memory = get_qdrant_memory() + print(f"✓ Initialized Tier 1 (Buffer) and Tier 2/3 (Qdrant)") + + # 1. Add turns to buffer (Tier 1) + turns = [ + ConversationTurn(role=MessageRole.USER, content="Hello!", turn_number=1), + ConversationTurn(role=MessageRole.ASSISTANT, content="Hi there! How can I help?", turn_number=2), + ConversationTurn(role=MessageRole.USER, content="How are you?", turn_number=3), + ConversationTurn(role=MessageRole.ASSISTANT, content="I'm doing great, thanks!", turn_number=4), + ] + + for turn in turns: + await buffer_memory.add_turn(test_conv_id, turn) + + # Verify buffer has them + buffer = await buffer_memory.get_buffer(test_conv_id) + assert len(buffer.turns) == 4 + print(f"✓ Tier 1 buffer: {len(buffer.turns)} turns stored") + + # 2. Move to Qdrant (Tier 2/3) - simulating consolidation + for turn in buffer.turns: + await qdrant_memory.add_turn(test_conv_id, turn) + + # Verify Qdrant has them + qdrant_turns = await qdrant_memory.get_turns(test_conv_id) + assert len(qdrant_turns) == 4 + print(f"✓ Tier 2/3 Qdrant: {len(qdrant_turns)} turns stored") + + # 3. Test semantic search across consolidated data + search_results = await qdrant_memory.similarity_search( + query="greeting hello", + conversation_id=test_conv_id, + limit=2 + ) + assert len(search_results) > 0 + print(f"✓ Semantic search: {len(search_results)} matches found") + print(f" Best match: '{search_results[0]['content']}' (score: {search_results[0]['score']:.3f})") + + # 4. Test data consistency + buffer_content = [t.content for t in buffer.turns] + qdrant_content = [t.content for t in qdrant_turns] + assert buffer_content == qdrant_content + print(f"✓ Data consistency verified (Buffer ↔ Qdrant)") + + # Cleanup + await qdrant_memory.clear_conversation(test_conv_id) + await buffer_memory.clear_conversation(test_conv_id) + print(f"✓ Cleanup complete") + + print("\n✅ Full Integration Tests: PASSED") + return True + + except Exception as e: + print(f"\n❌ Full Integration Tests: FAILED") + print(f"Error: {e}") + import traceback + traceback.print_exc() + + # Cleanup on error + try: + await qdrant_memory.clear_conversation(test_conv_id) + await buffer_memory.clear_conversation(test_conv_id) + except: + pass + + return False + + +def main(): + """Run all tests""" + print("\n" + "="*60) + print("PHASE 2 MEMORY SYSTEM - INTEGRATION TESTS") + print("="*60) + print(f"Start time: {datetime.utcnow().isoformat()}") + + results = [] + + # Test 1: Embedding Client + results.append(("Embedding Client", test_embedding_client())) + + # Test 2: Qdrant Memory + results.append(("Qdrant Memory", asyncio.run(test_qdrant_memory()))) + + # Test 3: Full Integration + results.append(("Full Integration", asyncio.run(test_full_integration()))) + + # Summary + print("\n" + "="*60) + print("TEST SUMMARY") + print("="*60) + + for test_name, passed in results: + status = "✅ PASSED" if passed else "❌ FAILED" + print(f"{test_name:.<40} {status}") + + total = len(results) + passed = sum(1 for _, p in results if p) + failed = total - passed + + print(f"\nTotal: {total} | Passed: {passed} | Failed: {failed}") + print(f"Success rate: {(passed/total)*100:.1f}%") + + if all(p for _, p in results): + print("\n" + "="*60) + print("🎉 ALL TESTS PASSED!") + print("="*60) + print("\nPhase 2 Memory System Status: ✅ FUNCTIONAL") + print("- Embedding client working (384d vectors)") + print("- Qdrant storage working (chronological + semantic)") + print("- Full integration working (Tier 1 ↔ Tier 2/3)") + return 0 + else: + print("\n" + "="*60) + print("❌ SOME TESTS FAILED") + print("="*60) + return 1 + + +if __name__ == "__main__": + exit(main())