Initial commit: scheduler service extraction from portainer-core

Extracted standalone scheduler service with: - FastAPI REST API for task management - APScheduler-based task execution - PostgreSQL persistence - Docker container support - Gitea Actions CI/CD workflow 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-11 11:59:32 +01:00
commit 64574bcc39
31 changed files with 6284 additions and 0 deletions
@@ -0,0 +1,530 @@
+"""
+The Scheduler - System-wide maintenance orchestration.
+Handles backups, documentation mirroring, cleanup, and automated tasks.
+
+Architecture: Hybrid APScheduler + DB-based priority system
+- APScheduler runs a single job every minute
+- Job queries DB for tasks scheduled in that minute
+- Executes up to 5 tasks concurrently based on priority
+"""
+from fastapi import FastAPI, HTTPException, Depends, Header
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.triggers.cron import CronTrigger
+from contextlib import asynccontextmanager
+import logging
+
+from src.config import get_settings, Settings
+from src.tasks.executor import TaskExecutor
+from src.models import TaskCreate, TaskUpdate, TaskResponse
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Global instances
+scheduler: AsyncIOScheduler | None = None
+task_executor: TaskExecutor | None = None
+
+
+def get_scheduler() -> AsyncIOScheduler:
+    """Dependency to get scheduler instance."""
+    if scheduler is None:
+        raise HTTPException(500, "Scheduler not initialized")
+    return scheduler
+
+
+def get_task_executor() -> TaskExecutor:
+    """Dependency to get task executor instance."""
+    if task_executor is None:
+        raise HTTPException(500, "Task executor not initialized")
+    return task_executor
+
+# Lifespan manager for startup/shutdown
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Manage application lifecycle - startup and shutdown."""
+    global scheduler, task_executor
+
+    settings = get_settings()
+
+    # Startup
+    logger.info("=" * 60)
+    logger.info("The Scheduler - System-wide Maintenance Orchestration")
+    logger.info("=" * 60)
+    logger.info(f"Architecture: Hybrid APScheduler + DB-based priority system")
+    logger.info(f"Database: {settings.postgres_host}:{settings.postgres_port}/{settings.postgres_db}")
+    logger.info(f"API: http://{settings.host}:{settings.port}")
+    logger.info(f"Docs: http://{settings.host}:{settings.port}/docs")
+    logger.info("=" * 60)
+
+    # Initialize task executor
+    task_executor = TaskExecutor(settings)
+    logger.info("Task executor initialized (max 5 concurrent tasks)")
+
+    # Initialize APScheduler with minimal configuration
+    # No jobstore needed - we only have one in-memory job
+    scheduler = AsyncIOScheduler(
+        job_defaults={
+            'coalesce': True,        # Combine missed runs
+            'max_instances': 1,      # Only one instance running
+            'misfire_grace_time': 30  # 30s grace period for minute-based execution
+        }
+    )
+
+    # Add the single minute-based task processor
+    scheduler.add_job(
+        func=task_executor.process_minute,
+        trigger=CronTrigger(minute='*'),  # Run every minute
+        id='process_tasks',
+        name='Process scheduled tasks',
+        replace_existing=True
+    )
+
+    try:
+        scheduler.start()
+        logger.info("Scheduler started - processing tasks every minute")
+        logger.info("Priority system: 1-5 (emergency/system), 10-30 (user), 40-70+ (maintenance)")
+    except Exception as e:
+        logger.error(f"Failed to start scheduler: {e}")
+        raise
+
+    yield
+
+    # Shutdown
+    logger.info("Stopping The Scheduler...")
+    if scheduler:
+        scheduler.shutdown(wait=True)
+    logger.info("Scheduler stopped")
+
+
+# FastAPI app
+app = FastAPI(
+    title="The Scheduler",
+    version="1.0.0",
+    description="System-wide maintenance orchestration - backups, doc mirroring, cleanup, task automation",
+    lifespan=lifespan
+)
+
+# Dependencies
+async def verify_api_key(
+    authorization: str = Header(None),
+    settings: Settings = Depends(get_settings)
+):
+    """Verify API key from Authorization header."""
+    if not authorization or not authorization.startswith("Bearer "):
+        raise HTTPException(401, "Missing API key")
+    key = authorization.replace("Bearer ", "")
+    if key != settings.scheduler_api_key:
+        raise HTTPException(403, "Invalid API key")
+    return key
+
+# ============================================================================
+# Public Endpoints (no auth required)
+# ============================================================================
+
+@app.get("/health")
+async def health(
+    settings: Settings = Depends(get_settings),
+    sched: AsyncIOScheduler = Depends(get_scheduler)
+):
+    """Health check endpoint."""
+    return {
+        "status": "healthy",
+        "scheduler_running": sched.running,
+        "jobs_count": len(sched.get_jobs()),
+        "database": settings.postgres_db
+    }
+
+# ============================================================================
+# Protected Endpoints (require API key)
+# ============================================================================
+
+@app.get("/tasks")
+async def list_tasks(
+    enabled: bool = None,
+    service: str = None,
+    api_key: str = Depends(verify_api_key),
+    executor: TaskExecutor = Depends(get_task_executor)
+):
+    """List all scheduled tasks from database."""
+    import psycopg2.extras
+
+    query = "SELECT * FROM scheduled_tasks WHERE 1=1"
+    params = []
+
+    if enabled is not None:
+        query += " AND enabled = %s"
+        params.append(enabled)
+
+    if service:
+        query += " AND service = %s"
+        params.append(service)
+
+    query += " ORDER BY priority ASC, task_name ASC"
+
+    with executor.get_db_connection() as conn:
+        with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+            cur.execute(query, params)
+            tasks = [dict(task) for task in cur.fetchall()]
+
+    return {
+        "tasks": tasks,
+        "count": len(tasks)
+    }
+
+@app.get("/tasks/{task_name}")
+async def get_task_details(
+    task_name: str,
+    api_key: str = Depends(verify_api_key),
+    executor: TaskExecutor = Depends(get_task_executor)
+):
+    """Get details for a specific task."""
+    import psycopg2.extras
+
+    with executor.get_db_connection() as conn:
+        with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+            cur.execute("SELECT * FROM scheduled_tasks WHERE task_name = %s", (task_name,))
+            task = cur.fetchone()
+
+    if not task:
+        raise HTTPException(404, f"Task '{task_name}' not found")
+
+    return dict(task)
+
+@app.post("/tasks", response_model=TaskResponse, tags=["Task Management"])
+async def create_task(
+    task: TaskCreate,
+    api_key: str = Depends(verify_api_key),
+    executor: TaskExecutor = Depends(get_task_executor)
+):
+    """
+    Create a new scheduled task.
+
+    ## Schedule Pattern
+
+    Use cron-style fields where `-1` means "every":
+    - `minute: -1, hour: -1` → Runs every minute
+    - `minute: 0, hour: -1` → Runs at minute 0 of every hour (hourly)
+    - `minute: 0, hour: 2` → Runs at 2:00 AM every day
+    - `minute: 0, hour: 2, day_of_week: 0` → Runs at 2:00 AM every Monday
+
+    ## Priority Levels
+
+    - **1-5**: Emergency/critical system tasks
+    - **10-30**: User-initiated tasks
+    - **40-70**: Background maintenance
+    - **70+**: Low priority cleanup
+
+    ## Executor Types
+
+    ### `rest_api` Executor
+
+    Calls HTTP endpoints. Config format:
+    ```json
+    {
+      "method": "POST",
+      "url": "http://service:port/endpoint",
+      "headers": {
+        "Authorization": "Bearer ${ENV_VAR}",
+        "Content-Type": "application/json"
+      },
+      "body": {"key": "value"}
+    }
+    ```
+
+    Environment variables can be referenced with `${VAR_NAME}` syntax.
+
+    ## Example: Librarian Consolidation Task
+
+    ```json
+    {
+      "task_name": "librarian_consolidation",
+      "service": "library-desk",
+      "executor": "rest_api",
+      "priority": 25,
+      "description": "Processes search queries and consolidates knowledge",
+      "minute": 0,
+      "hour": -1,
+      "config": {
+        "method": "POST",
+        "url": "http://library-desk:8089/consolidation/run",
+        "headers": {
+          "Authorization": "Bearer ${LIBRARY_DESK_API_KEY}",
+          "Content-Type": "application/json"
+        },
+        "body": {
+          "dry_run": false,
+          "process_limit": 10
+        }
+      }
+    }
+    ```
+
+    This creates an hourly task that calls the consolidation endpoint.
+    """
+    import psycopg2.extras
+    import json
+
+    # Convert task model to dict
+    task_data = task.model_dump()
+
+    # Convert config dict to JSON string if present
+    if 'config' in task_data and isinstance(task_data['config'], dict):
+        task_data['config'] = json.dumps(task_data['config'])
+
+    with executor.get_db_connection() as conn:
+        with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+            cur.execute("""
+                INSERT INTO scheduled_tasks
+                    (task_name, service, executor, priority, minute, hour,
+                     day_of_month, month, day_of_week, enabled, description,
+                     config, max_retries, timeout_seconds, created_by)
+                VALUES
+                    (%(task_name)s, %(service)s, %(executor)s, %(priority)s,
+                     %(minute)s, %(hour)s, %(day_of_month)s, %(month)s,
+                     %(day_of_week)s, %(enabled)s, %(description)s,
+                     %(config)s::jsonb, %(max_retries)s, %(timeout_seconds)s,
+                     %(created_by)s)
+                RETURNING *
+            """, task_data)
+            new_task = dict(cur.fetchone())
+            conn.commit()
+
+    logger.info(f"Created task: {new_task['task_name']}")
+    return new_task
+
+@app.put("/tasks/{task_name}")
+async def update_task(
+    task_name: str,
+    task_data: dict,
+    api_key: str = Depends(verify_api_key),
+    executor: TaskExecutor = Depends(get_task_executor)
+):
+    """Update an existing scheduled task."""
+    import psycopg2.extras
+    import json
+
+    # Build update query dynamically
+    allowed_fields = ['service', 'executor', 'priority', 'minute', 'hour',
+                     'day_of_month', 'month', 'day_of_week', 'enabled',
+                     'description', 'config', 'max_retries', 'timeout_seconds']
+
+    updates = {k: v for k, v in task_data.items() if k in allowed_fields}
+    if not updates:
+        raise HTTPException(400, "No valid fields to update")
+
+    # Convert config dict to JSON string if present
+    if 'config' in updates and isinstance(updates['config'], dict):
+        updates['config'] = json.dumps(updates['config'])
+
+    set_clause = ', '.join([f"{k} = %({k})s" for k in updates.keys()])
+
+    with executor.get_db_connection() as conn:
+        with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+            cur.execute(f"""
+                UPDATE scheduled_tasks
+                SET {set_clause}, updated_at = NOW()
+                WHERE task_name = %(task_name)s
+                RETURNING *
+            """, {**updates, 'task_name': task_name})
+
+            updated_task = cur.fetchone()
+            if not updated_task:
+                raise HTTPException(404, f"Task '{task_name}' not found")
+
+            conn.commit()
+
+    logger.info(f"Updated task: {task_name}")
+    return dict(updated_task)
+
+@app.delete("/tasks/{task_name}")
+async def delete_task(
+    task_name: str,
+    api_key: str = Depends(verify_api_key),
+    executor: TaskExecutor = Depends(get_task_executor)
+):
+    """Delete a scheduled task."""
+    with executor.get_db_connection() as conn:
+        with conn.cursor() as cur:
+            cur.execute("""
+                DELETE FROM scheduled_tasks
+                WHERE task_name = %s
+                RETURNING task_name
+            """, (task_name,))
+
+            deleted = cur.fetchone()
+            if not deleted:
+                raise HTTPException(404, f"Task '{task_name}' not found")
+
+            conn.commit()
+
+    logger.info(f"Deleted task: {task_name}")
+    return {"message": f"Task '{task_name}' deleted successfully"}
+
+@app.post("/tasks/{task_name}/trigger")
+async def trigger_task(
+    task_name: str,
+    api_key: str = Depends(verify_api_key),
+    executor: TaskExecutor = Depends(get_task_executor)
+):
+    """Manually trigger a task to run immediately."""
+    import psycopg2.extras
+
+    # Get task details
+    with executor.get_db_connection() as conn:
+        with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+            cur.execute("SELECT * FROM scheduled_tasks WHERE task_name = %s", (task_name,))
+            task = cur.fetchone()
+
+    if not task:
+        raise HTTPException(404, f"Task '{task_name}' not found")
+
+    if not task['enabled']:
+        raise HTTPException(400, f"Task '{task_name}' is disabled")
+
+    # Execute task immediately in background
+    import asyncio
+    asyncio.create_task(executor.execute_task(dict(task)))
+
+    logger.info(f"Manually triggered task: {task_name}")
+    return {
+        "message": f"Task '{task_name}' triggered successfully",
+        "task_name": task_name,
+        "priority": task['priority'],
+        "executor": task['executor']
+    }
+
+# Legacy endpoints (deprecated)
+@app.post("/tasks/backup")
+async def trigger_backup(api_key: str = Depends(verify_api_key)):
+    """Trigger backup tasks manually (deprecated - use POST /tasks/{name}/trigger)"""
+    # TODO: Implement backup executor
+    logger.info("Manual backup triggered")
+    return {
+        "message": "Backup task triggered",
+        "status": "not_implemented",
+        "note": "Backup executor needs to be implemented"
+    }
+
+@app.post("/tasks/docs/update")
+async def trigger_docs_update(
+    project: str = None,
+    api_key: str = Depends(verify_api_key)
+):
+    """Trigger documentation mirror update"""
+    # TODO: Implement doc mirror executor
+    logger.info(f"Doc mirror update triggered for project: {project or 'all'}")
+    return {
+        "message": f"Documentation update triggered for {project or 'all projects'}",
+        "status": "not_implemented",
+        "note": "Doc mirror executor needs to be implemented"
+    }
+
+@app.post("/tasks/docs/check-versions")
+async def check_doc_versions(api_key: str = Depends(verify_api_key)):
+    """Check for new documentation versions"""
+    # TODO: Implement version check executor
+    logger.info("Version check triggered")
+    return {
+        "message": "Version check triggered",
+        "status": "not_implemented",
+        "note": "Version check executor needs to be implemented"
+    }
+
+@app.post("/tasks/cleanup")
+async def trigger_cleanup(api_key: str = Depends(verify_api_key)):
+    """Run cleanup tasks"""
+    # TODO: Implement cleanup executor
+    logger.info("Cleanup task triggered")
+    return {
+        "message": "Cleanup task triggered",
+        "status": "not_implemented",
+        "note": "Cleanup executor needs to be implemented"
+    }
+
+@app.get("/executions")
+async def task_history(
+    limit: int = 20,
+    task_name: str = None,
+    service: str = None,
+    status: str = None,
+    api_key: str = Depends(verify_api_key),
+    executor: TaskExecutor = Depends(get_task_executor)
+):
+    """View task execution history."""
+    import psycopg2.extras
+
+    query = "SELECT * FROM task_executions WHERE 1=1"
+    params = []
+
+    if task_name:
+        query += " AND task_name = %s"
+        params.append(task_name)
+
+    if service:
+        query += " AND service = %s"
+        params.append(service)
+
+    if status:
+        query += " AND status = %s"
+        params.append(status)
+
+    query += " ORDER BY triggered_at DESC LIMIT %s"
+    params.append(limit)
+
+    with executor.get_db_connection() as conn:
+        with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+            cur.execute(query, params)
+            executions = [dict(ex) for ex in cur.fetchall()]
+
+    return {
+        "executions": executions,
+        "count": len(executions),
+        "limit": limit
+    }
+
+@app.get("/stats")
+async def stats(
+    api_key: str = Depends(verify_api_key),
+    settings: Settings = Depends(get_settings),
+    sched: AsyncIOScheduler = Depends(get_scheduler),
+    task_exec: TaskExecutor = Depends(get_task_executor)
+):
+    """Get system statistics."""
+    import psycopg2.extras
+
+    # Query task stats from database
+    with task_exec.get_db_connection() as conn:
+        with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+            # Count enabled tasks
+            cur.execute("SELECT COUNT(*) as count FROM scheduled_tasks WHERE enabled = true")
+            enabled_tasks = cur.fetchone()['count']
+
+            # Count running tasks
+            cur.execute("SELECT COUNT(*) as count FROM task_executions WHERE status = 'running'")
+            running_tasks = cur.fetchone()['count']
+
+            # Recent execution stats (last 24 hours)
+            cur.execute("""
+                SELECT status, COUNT(*) as count
+                FROM task_executions
+                WHERE triggered_at > NOW() - INTERVAL '24 hours'
+                GROUP BY status
+            """)
+            execution_stats = {row['status']: row['count'] for row in cur.fetchall()}
+
+    return {
+        "scheduler_running": sched.running,
+        "minute_processor_active": True,  # If we got here, it's running
+        "database": settings.postgres_db,
+        "tasks_enabled": enabled_tasks,
+        "tasks_currently_running": running_tasks,
+        "concurrent_limit": 5,
+        "execution_stats_24h": execution_stats,
+        "priority_system": "1-5 (emergency/system), 10-30 (user), 40-70+ (maintenance)"
+    }
+