scheduler/src/executors/portainer_backup_executor.py

"""
Portainer Backup Executor

Archives Portainer's own state through its `/api/backup` endpoint.

Why it needs backing up separately: Portainer keeps every stack definition,
endpoint, user and access-control rule in a BoltDB inside the Docker volume
`portainer_data`, which lives under /var/lib/docker/volumes/. The daily config
backup covers ~/docker-data and code-server-config only, so that volume is not
in it. Losing it takes all 24 stack definitions with it.

Why the API rather than tarring the volume: BoltDB is a single memory-mapped
file, so copying it while Portainer is writing can capture a torn page. The API
serialises a consistent snapshot.

The archive contains TLS certificates and private keys, so it is written 0600.

Config schema:
    {
        "url": "http://172.17.0.1:8001",     # Portainer is host-networked, so a
                                             # container name does not resolve;
                                             # use the bridge gateway
        "api_key": "${PORTAINER_API_KEY}",   # ${VAR} reads the container env
        "output_dir": "/backups/portainer",
        "retention_days": 30,
        "password": ""                       # optional; encrypts the archive
    }
"""
import logging
import os
import re
import tarfile
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path

import httpx

from src.config import Settings
from src.executors import health_report

logger = logging.getLogger(__name__)

BACKUP_TIMEOUT = 300
FILENAME_RE = re.compile(r"^portainer-\d{8}T\d{6}Z\.tar\.gz$")


def _substitute_env(value: str) -> str:
    """Expand ${VAR} against the container environment, as rest_api does."""
    if not isinstance(value, str):
        return value
    for var in re.findall(r"\$\{([A-Z_][A-Z0-9_]*)\}", value):
        resolved = os.getenv(var, "")
        if not resolved:
            logger.warning("environment variable not found: %s", var)
        value = value.replace(f"${{{var}}}", resolved)
    return value


def _prune(output_dir: Path, retention_days: int) -> int:
    """Delete archives older than the retention window. Returns how many went."""
    cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
    removed = 0
    for path in output_dir.glob("portainer-*.tar.gz"):
        # Match the exact name this executor writes; never delete a stray file
        # someone else put here.
        if not FILENAME_RE.match(path.name):
            continue
        if datetime.fromtimestamp(path.stat().st_mtime, timezone.utc) < cutoff:
            path.unlink()
            removed += 1
            logger.info("pruned old portainer backup: %s", path.name)
    return removed


async def _run(config: dict, settings: Settings) -> str:
    url = _substitute_env(config.get("url", "")).rstrip("/")
    api_key = _substitute_env(config.get("api_key", ""))
    output_dir = Path(config.get("output_dir", "/backups/portainer"))
    retention_days = config.get("retention_days", 30)
    password = _substitute_env(config.get("password", "") or "")

    if not url:
        raise ValueError("Missing required config: 'url'")
    if not api_key:
        raise ValueError("Missing or unresolved config: 'api_key'")
    if not isinstance(retention_days, int) or isinstance(retention_days, bool) or retention_days < 1:
        raise ValueError(f"retention_days must be a positive integer, got {retention_days!r}")

    output_dir.mkdir(parents=True, exist_ok=True)
    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
    final = output_dir / f"portainer-{stamp}.tar.gz"
    partial = final.with_suffix(".partial")

    started = time.monotonic()
    try:
        async with httpx.AsyncClient(timeout=BACKUP_TIMEOUT) as client:
            response = await client.post(
                f"{url}/api/backup",
                headers={"X-API-Key": api_key, "Content-Type": "application/json"},
                json={"password": password} if password else {},
            )
            if response.status_code != 200:
                raise Exception(
                    f"Portainer returned HTTP {response.status_code}: {response.text[:200]}"
                )
            partial.write_bytes(response.content)

        # A 200 with a truncated body is still a failed backup. An archive that
        # cannot be opened is worse than a missing one, because it looks like a
        # backup until the day it is needed.
        if not password:
            try:
                with tarfile.open(partial, "r:gz") as archive:
                    entries = len(archive.getnames())
            except Exception as exc:  # noqa: BLE001
                # Deliberately broad. A truncated archive raises EOFError, which
                # is neither TarError nor OSError, and any failure to open it
                # means the same thing regardless of type: this is not a backup.
                raise Exception(f"response is not a readable archive: {exc}") from exc
        else:
            entries = -1  # encrypted; contents cannot be verified here

        partial.replace(final)
        final.chmod(0o600)  # contains TLS certs and private keys
    finally:
        if partial.exists():
            partial.unlink()

    removed = _prune(output_dir, retention_days)
    kept = len([p for p in output_dir.glob("portainer-*.tar.gz") if FILENAME_RE.match(p.name)])
    size_mb = final.stat().st_size / 1_048_576
    elapsed = time.monotonic() - started

    summary = (
        f"backed up Portainer to {final.name} "
        f"({size_mb:.2f} MB{'' if entries < 0 else f', {entries} entries'}, {elapsed:.1f}s); "
        f"kept {kept}, pruned {removed} older than {retention_days}d"
    )
    logger.info(summary)
    return summary


async def execute(config: dict, settings: Settings) -> str:
    """Run the backup and report its own outcome to check_history (D-33, T-69).

    The report wraps the work rather than living inside it, so the failure path
    cannot be forgotten: an exception is reported as critical and then re-raised,
    leaving the task's own status untouched. Reporting only success would
    reproduce exactly the blind spot this replaces — a monitor that cannot tell
    a failed backup from one that has not run.
    """
    try:
        output = await _run(config, settings)
    except Exception as exc:
        await health_report.report_async(
            settings,
            domain="backup",
            status=health_report.CRITICAL,
            source="scheduler/portainer_backup_executor",
            summary=f"backup failed: {str(exc)[:300]}",
            metrics={"job": "scheduler/portainer_backup_executor", "error": str(exc)[:400]},
        )
        raise
    await health_report.report_async(
        settings,
        domain="backup",
        status=health_report.OK,
        source="scheduler/portainer_backup_executor",
        summary=output[:400],
        metrics={"job": "scheduler/portainer_backup_executor"},
    )
    return output