feat(executors): back up Portainer's own state

Portainer keeps every stack definition, endpoint, user and access-control
rule in a BoltDB inside the portainer_data Docker volume. That volume
sits under /var/lib/docker/volumes/, and the daily config backup covers
~/docker-data and code-server-config only — so the thing that defines all
24 stacks was the one thing not backed up.

Calls Portainer's /api/backup rather than tarring the volume. BoltDB is a
single memory-mapped file, so copying it while Portainer writes can
capture a torn page; the API serialises a consistent snapshot.

A 200 whose body is not a readable archive is treated as failure. An
archive that will not open is worse than a missing one, because it looks
like a backup until the day it is needed. Writing that check found a real
gap in it: a truncated tar.gz raises EOFError, which is neither TarError
nor OSError, so the first version of the guard let it through.

Archives contain TLS certificates and private keys and are written 0600.
Retention only ever deletes files matching the exact name this executor
writes, so an unrelated archive left in the same directory survives.

Co-Authored-By: Claude <noreply@anthropic.com>

TASK_REGISTRATION.md

@@ -114,6 +114,7 @@ modules that actually exist:
 - `gitea_release_cleanup_executor`: drop old Gitea releases, keeping the newest N
 - `postgres_retention_executor`: delete rows past a retention window (see below)
 - `docker_prune_executor`: reclaim Docker disk usage (see below)
+- `portainer_backup_executor`: archive Portainer's own state via its backup API (see below)
 - `example_executor`: demo/test
 
 There is **no `shell` or `python` executor**. Earlier revisions of this document
@@ -137,6 +138,31 @@ identifier pattern because they cannot be bound as query parameters.
 }
 ```
 
+#### `portainer_backup_executor`
+
+Portainer keeps every stack definition, endpoint, user and access-control rule in
+a BoltDB inside the `portainer_data` Docker volume, which lives under
+`/var/lib/docker/volumes/` and is **not** covered by the daily config backup.
+This calls Portainer's `/api/backup` rather than tarring the volume: BoltDB is a
+single memory-mapped file, so copying it live can capture a torn page.
+
+The archive contains TLS certificates and private keys and is written `0600`. A
+200 response whose body is not a readable archive is treated as a failure — an
+archive that will not open is worse than a missing one, because it looks like a
+backup until the day it is needed.
+
+```json
+{
+  "url": "${PORTAINER_URL}",
+  "api_key": "${PORTAINER_API_KEY}",
+  "output_dir": "/backups/portainer",
+  "retention_days": 30
+}
+```
+
+Portainer runs host-networked, so a container name does not resolve; use the
+host address. Requires `/mnt/media/backups/portainer` mounted into the container.
+
 #### `docker_prune_executor`
 
 Uses the docker socket already mounted into the container. Only the two stages

src/executors/portainer_backup_executor.py

@@ -0,0 +1,140 @@
+"""
+Portainer Backup Executor
+
+Archives Portainer's own state through its `/api/backup` endpoint.
+
+Why it needs backing up separately: Portainer keeps every stack definition,
+endpoint, user and access-control rule in a BoltDB inside the Docker volume
+`portainer_data`, which lives under /var/lib/docker/volumes/. The daily config
+backup covers ~/docker-data and code-server-config only, so that volume is not
+in it. Losing it takes all 24 stack definitions with it.
+
+Why the API rather than tarring the volume: BoltDB is a single memory-mapped
+file, so copying it while Portainer is writing can capture a torn page. The API
+serialises a consistent snapshot.
+
+The archive contains TLS certificates and private keys, so it is written 0600.
+
+Config schema:
+    {
+        "url": "http://172.17.0.1:8001",     # Portainer is host-networked, so a
+                                             # container name does not resolve;
+                                             # use the bridge gateway
+        "api_key": "${PORTAINER_API_KEY}",   # ${VAR} reads the container env
+        "output_dir": "/backups/portainer",
+        "retention_days": 30,
+        "password": ""                       # optional; encrypts the archive
+    }
+"""
+import logging
+import os
+import re
+import tarfile
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import httpx
+
+from src.config import Settings
+
+logger = logging.getLogger(__name__)
+
+BACKUP_TIMEOUT = 300
+FILENAME_RE = re.compile(r"^portainer-\d{8}T\d{6}Z\.tar\.gz$")
+
+
+def _substitute_env(value: str) -> str:
+    """Expand ${VAR} against the container environment, as rest_api does."""
+    if not isinstance(value, str):
+        return value
+    for var in re.findall(r"\$\{([A-Z_][A-Z0-9_]*)\}", value):
+        resolved = os.getenv(var, "")
+        if not resolved:
+            logger.warning("environment variable not found: %s", var)
+        value = value.replace(f"${{{var}}}", resolved)
+    return value
+
+
+def _prune(output_dir: Path, retention_days: int) -> int:
+    """Delete archives older than the retention window. Returns how many went."""
+    cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
+    removed = 0
+    for path in output_dir.glob("portainer-*.tar.gz"):
+        # Match the exact name this executor writes; never delete a stray file
+        # someone else put here.
+        if not FILENAME_RE.match(path.name):
+            continue
+        if datetime.fromtimestamp(path.stat().st_mtime, timezone.utc) < cutoff:
+            path.unlink()
+            removed += 1
+            logger.info("pruned old portainer backup: %s", path.name)
+    return removed
+
+
+async def execute(config: dict, settings: Settings) -> str:
+    url = _substitute_env(config.get("url", "")).rstrip("/")
+    api_key = _substitute_env(config.get("api_key", ""))
+    output_dir = Path(config.get("output_dir", "/backups/portainer"))
+    retention_days = config.get("retention_days", 30)
+    password = _substitute_env(config.get("password", "") or "")
+
+    if not url:
+        raise ValueError("Missing required config: 'url'")
+    if not api_key:
+        raise ValueError("Missing or unresolved config: 'api_key'")
+    if not isinstance(retention_days, int) or isinstance(retention_days, bool) or retention_days < 1:
+        raise ValueError(f"retention_days must be a positive integer, got {retention_days!r}")
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    final = output_dir / f"portainer-{stamp}.tar.gz"
+    partial = final.with_suffix(".partial")
+
+    started = time.monotonic()
+    try:
+        async with httpx.AsyncClient(timeout=BACKUP_TIMEOUT) as client:
+            response = await client.post(
+                f"{url}/api/backup",
+                headers={"X-API-Key": api_key, "Content-Type": "application/json"},
+                json={"password": password} if password else {},
+            )
+            if response.status_code != 200:
+                raise Exception(
+                    f"Portainer returned HTTP {response.status_code}: {response.text[:200]}"
+                )
+            partial.write_bytes(response.content)
+
+        # A 200 with a truncated body is still a failed backup. An archive that
+        # cannot be opened is worse than a missing one, because it looks like a
+        # backup until the day it is needed.
+        if not password:
+            try:
+                with tarfile.open(partial, "r:gz") as archive:
+                    entries = len(archive.getnames())
+            except Exception as exc:  # noqa: BLE001
+                # Deliberately broad. A truncated archive raises EOFError, which
+                # is neither TarError nor OSError, and any failure to open it
+                # means the same thing regardless of type: this is not a backup.
+                raise Exception(f"response is not a readable archive: {exc}") from exc
+        else:
+            entries = -1  # encrypted; contents cannot be verified here
+
+        partial.replace(final)
+        final.chmod(0o600)  # contains TLS certs and private keys
+    finally:
+        if partial.exists():
+            partial.unlink()
+
+    removed = _prune(output_dir, retention_days)
+    kept = len([p for p in output_dir.glob("portainer-*.tar.gz") if FILENAME_RE.match(p.name)])
+    size_mb = final.stat().st_size / 1_048_576
+    elapsed = time.monotonic() - started
+
+    summary = (
+        f"backed up Portainer to {final.name} "
+        f"({size_mb:.2f} MB{'' if entries < 0 else f', {entries} entries'}, {elapsed:.1f}s); "
+        f"kept {kept}, pruned {removed} older than {retention_days}d"
+    )
+    logger.info(summary)
+    return summary

tests/test_portainer_backup_executor.py

@@ -0,0 +1,175 @@
+"""
+Tests for the Portainer backup executor.
+
+The point of this executor is producing an archive that will still open on the
+day it is needed, so most of these cover the failure paths: a truncated body
+behind a 200, a partial file left on disk, and retention deleting the wrong
+thing.
+"""
+import gzip
+import io
+import tarfile
+from datetime import datetime, timedelta, timezone
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from src.config import Settings
+from src.executors import portainer_backup_executor as pbe
+
+
+def _tar_gz_bytes(names=("compose/1/docker-compose.yml", "certs/cert.pem")) -> bytes:
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+        for n in names:
+            data = b"x"
+            info = tarfile.TarInfo(name=n)
+            info.size = len(data)
+            tar.addfile(info, io.BytesIO(data))
+    return buf.getvalue()
+
+
+def _mock_post(status=200, content=None):
+    response = MagicMock()
+    response.status_code = status
+    response.content = content if content is not None else _tar_gz_bytes()
+    response.text = "error body"
+    client = MagicMock()
+    client.post = AsyncMock(return_value=response)
+    ctx = MagicMock()
+    ctx.__aenter__ = AsyncMock(return_value=client)
+    ctx.__aexit__ = AsyncMock(return_value=False)
+    return ctx, client
+
+
+@pytest.mark.executor
+@pytest.mark.unit
+class TestConfigValidation:
+
+    @pytest.mark.asyncio
+    async def test_missing_url_rejected(self, test_settings: Settings, tmp_path):
+        with pytest.raises(ValueError, match="url"):
+            await pbe.execute({"api_key": "k", "output_dir": str(tmp_path)}, test_settings)
+
+    @pytest.mark.asyncio
+    async def test_missing_api_key_rejected(self, test_settings: Settings, tmp_path):
+        with pytest.raises(ValueError, match="api_key"):
+            await pbe.execute({"url": "http://x", "output_dir": str(tmp_path)}, test_settings)
+
+    @pytest.mark.asyncio
+    async def test_unresolved_env_var_rejected(self, test_settings: Settings, tmp_path, monkeypatch):
+        """${VAR} that expands to nothing must fail, not send an empty key."""
+        monkeypatch.delenv("NOPE_MISSING", raising=False)
+        with pytest.raises(ValueError, match="api_key"):
+            await pbe.execute(
+                {"url": "http://x", "api_key": "${NOPE_MISSING}", "output_dir": str(tmp_path)},
+                test_settings,
+            )
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("bad", [0, -1, "30", None, True])
+    async def test_bad_retention_rejected(self, bad, test_settings: Settings, tmp_path):
+        with pytest.raises(ValueError, match="retention_days"):
+            await pbe.execute(
+                {"url": "http://x", "api_key": "k", "output_dir": str(tmp_path),
+                 "retention_days": bad},
+                test_settings,
+            )
+
+
+@pytest.mark.executor
+@pytest.mark.unit
+class TestBackupBehaviour:
+
+    def _config(self, tmp_path, **over):
+        cfg = {"url": "http://portainer:9000", "api_key": "k",
+               "output_dir": str(tmp_path), "retention_days": 30}
+        cfg.update(over)
+        return cfg
+
+    @pytest.mark.asyncio
+    async def test_writes_verified_archive(self, test_settings: Settings, tmp_path):
+        ctx, _ = _mock_post()
+        with patch("httpx.AsyncClient", return_value=ctx):
+            result = await pbe.execute(self._config(tmp_path), test_settings)
+
+        files = list(tmp_path.glob("portainer-*.tar.gz"))
+        assert len(files) == 1
+        assert "2 entries" in result
+        with tarfile.open(files[0], "r:gz") as tar:      # opens = usable backup
+            assert "certs/cert.pem" in tar.getnames()
+
+    @pytest.mark.asyncio
+    async def test_archive_is_not_world_readable(self, test_settings: Settings, tmp_path):
+        """It contains TLS private keys."""
+        ctx, _ = _mock_post()
+        with patch("httpx.AsyncClient", return_value=ctx):
+            await pbe.execute(self._config(tmp_path), test_settings)
+        f = next(tmp_path.glob("portainer-*.tar.gz"))
+        assert oct(f.stat().st_mode)[-3:] == "600"
+
+    @pytest.mark.asyncio
+    async def test_http_error_raises_and_leaves_nothing(self, test_settings: Settings, tmp_path):
+        ctx, _ = _mock_post(status=401)
+        with patch("httpx.AsyncClient", return_value=ctx):
+            with pytest.raises(Exception, match="HTTP 401"):
+                await pbe.execute(self._config(tmp_path), test_settings)
+        assert list(tmp_path.iterdir()) == []
+
+    @pytest.mark.asyncio
+    async def test_truncated_body_behind_200_is_rejected(self, test_settings: Settings, tmp_path):
+        """The dangerous case: a 200 whose body is not a usable archive."""
+        broken = _tar_gz_bytes()[:40]
+        ctx, _ = _mock_post(content=broken)
+        with patch("httpx.AsyncClient", return_value=ctx):
+            with pytest.raises(Exception, match="not a readable archive"):
+                await pbe.execute(self._config(tmp_path), test_settings)
+        # no .partial and no final file left behind
+        assert list(tmp_path.iterdir()) == []
+
+    @pytest.mark.asyncio
+    async def test_gzip_that_is_not_a_tar_is_rejected(self, test_settings: Settings, tmp_path):
+        ctx, _ = _mock_post(content=gzip.compress(b"not a tar"))
+        with patch("httpx.AsyncClient", return_value=ctx):
+            with pytest.raises(Exception, match="not a readable archive"):
+                await pbe.execute(self._config(tmp_path), test_settings)
+        assert list(tmp_path.iterdir()) == []
+
+    @pytest.mark.asyncio
+    async def test_api_key_resolved_from_env(self, test_settings: Settings, tmp_path, monkeypatch):
+        monkeypatch.setenv("PT_KEY", "secret-value")
+        ctx, client = _mock_post()
+        with patch("httpx.AsyncClient", return_value=ctx):
+            await pbe.execute(self._config(tmp_path, api_key="${PT_KEY}"), test_settings)
+        assert client.post.call_args.kwargs["headers"]["X-API-Key"] == "secret-value"
+
+
+@pytest.mark.executor
+@pytest.mark.unit
+class TestRetention:
+
+    def _age(self, path, days):
+        import os
+        old = (datetime.now(timezone.utc) - timedelta(days=days)).timestamp()
+        os.utime(path, (old, old))
+
+    def test_prunes_only_past_the_window(self, tmp_path):
+        fresh = tmp_path / "portainer-20260808T120000Z.tar.gz"
+        stale = tmp_path / "portainer-20260101T120000Z.tar.gz"
+        for f in (fresh, stale):
+            f.write_bytes(b"x")
+        self._age(stale, 45)
+
+        assert pbe._prune(tmp_path, 30) == 1
+        assert fresh.exists() and not stale.exists()
+
+    def test_leaves_unrelated_files_alone(self, tmp_path):
+        """Retention must not touch anything it did not write."""
+        other = tmp_path / "important-database-dump.tar.gz"
+        named_alike = tmp_path / "portainer-backup-manual.tar.gz"
+        for f in (other, named_alike):
+            f.write_bytes(b"x")
+            self._age(f, 400)
+
+        assert pbe._prune(tmp_path, 30) == 0
+        assert other.exists() and named_alike.exists()