feat(backup): add GCS offsite backup executor

Add gcs_backup_executor with git_bundle mode for backing up bare git
repos to Google Cloud Storage. Includes retention management and
bundle verification. Adds google-cloud-storage dependency and
GCS_CREDENTIALS_FILE setting.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.gitea/workflows/build.yml

@@ -1,12 +1,25 @@
 name: Build and Push
 
 on:
-  release:
+  push:
-    types: [published]
+    tags:
+      - 'v*'
 
 jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create Gitea Release
+        run: |
+          curl -sf -X POST \
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '{"tag_name": "${{ github.ref_name }}", "name": "Release ${{ github.ref_name }}", "body": "Automated release for ${{ github.ref_name }}"}' \
+            "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases"
+
   build:
     runs-on: ubuntu-latest
+    needs: release
     steps:
       - uses: actions/checkout@v4
 
@@ -25,3 +38,9 @@ jobs:
           tags: |
             git.schweitz.internal/jpmschweitzer/scheduler:latest
             git.schweitz.internal/jpmschweitzer/scheduler:${{ github.ref_name }}
+
+      - name: Trigger Watchtower update
+        if: success()
+        run: |
+          curl -sf -H "Authorization: Bearer ${{ secrets.WATCHTOWER_TOKEN }}" \
+            http://watchtower:8080/v1/update

AGENTS.md

@@ -0,0 +1,72 @@
+
+# AGENTS.md
+
+> **Start every session by reading this file.**
+> This file outlines the operational protocols, coding standards, and architectural decisions for this FastAPI project.
+
+## 1. Agent Operational Protocols
+
+### 🧠 Work Patterns (Plan-Act-Reflect)
+* **Plan:** Before writing code, briefly outline your plan. Identify which files you will touch and what the side effects might be.
+* **Act:** Execute the changes in small, atomic steps.
+* **Reflect:** After coding, verify your work. Did you break existing tests? Did you add new tests?
+
+### 🛡️ Git Discipline
+* **NEVER commit to `main` or `master` directly.** Always create a feature branch: `feature/your-feature-name` or `fix/issue-description`.
+* **Commit Messages:** Use the [Conventional Commits](https://www.conventionalcommits.org/) format.
+    * `feat: add user login endpoint`
+    * `fix: resolve database connection timeout`
+    * `refactor: split monolith dependency file`
+* **Atomic Commits:** Keep commits small. One logical change = one commit.
+
+### 📝 Changelog Maintenance
+* **Update `CHANGELOG.md`** with every user-facing change.
+* Format: `## [Unreleased] - YYYY-MM-DD` followed by `### Added`, `### Changed`, or `### Fixed`.
+
+### 🚀 Release Flow
+When changes are ready for deployment:
+
+1. **Ask user if deploy cycle is desired **
+
+2. **Update version** in `pyproject.toml`:
+   - Bug fixes: bump patch version (1.8.3 → 1.8.4)
+   - New features: bump minor version (1.8.4 → 1.9.0)
+
+3. **Update CHANGELOG.md**:
+   - Move items from `[Unreleased]` to new version section
+   - Add release date: `## [1.8.4] - 2025-12-16`
+
+4. **Commit and tag**:
+   ```bash
+   git add -A
+   git commit -m "fix: description of changes"
+   git tag v1.8.4
+   git push origin main --tags
+   ```
+
+5. **CI/CD triggers automatically**:
+   - Gitea CI builds Docker image on new tag
+   - Watchtower pulls and deploys to production
+   - Verify deployment: `curl http://192.168.86.149:8000/health`
+
+---
+
+## 2. FastAPI Architecture & Best Practices
+*Reference: [FastAPI Best Practices](https://github.com/zhanymkanov/fastapi-best-practices)*
+
+### 📂 Project Structure (Directory-based, NOT File-type based)
+Do **not** group files by type (e.g., one huge `routers` folder). Group by **domain/module** inside a `src/` directory.
+
+**Correct Structure:**
+```text
+src/
+├── auth/
+│   ├── router.py      # Endpoints
+│   ├── schemas.py     # Pydantic models
+│   ├── service.py     # Business logic (CRUD, etc.)
+│   ├── dependencies.py# Module-specific dependencies
+│   └── config.py      # Module-specific settings
+├── posts/
+│   ├── router.py
+│   └── ...
+└── main.py            # App entry point

CHANGELOG.md

@@ -4,6 +4,42 @@ All notable changes to The Scheduler will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.1.3] - 2026-01-08
+
+### Changed
+- Test release to validate CI/CD auto-deploy workflow
+
+## [1.1.2] - 2026-01-03
+
+### Fixed
+- CI: Use curl for release creation (release-action requires Go)
+
+## [1.1.1] - 2026-01-03
+
+### Changed
+- CI: Auto-create Gitea release on version tag push (v*) instead of manual release trigger
+
+## [1.1.0] - 2025-12-14
+
+### Added
+- **Gitea Release Cleanup Executor** (`gitea_release_cleanup_executor.py`)
+  - Automatically cleans up old releases across all Gitea repositories
+  - Configurable retention count (default: 5 releases per repo)
+  - Repository exclusion list support
+  - Dry-run mode for safe testing
+  - Designed to run before Watchtower to prevent image tag accumulation
+- **GITEA_TOKEN setting** in config for API token authentication (separate from password)
+
+## [1.0.4] - 2025-12-14
+
+### Fixed
+- CI/CD: Correct Watchtower port (8080)
+
+## [1.0.3] - 2025-12-14
+
+### Added
+- CI/CD: Trigger Watchtower update after successful Docker build
+
 ## [1.0.2] - 2025-12-14
 
 ### Fixed

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "the-scheduler"
-version = "1.0.2"
+version = "1.2.0"
 description = "System-wide maintenance orchestration - backups, doc mirroring, cleanup, task automation"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -26,6 +26,8 @@ dependencies = [
     "gitpython~=3.1.43",
     # Security/Auth
     "python-jose[cryptography]~=3.3.0",
+    # Cloud storage
+    "google-cloud-storage~=2.18.0",
 ]
 
 [project.optional-dependencies]

requirements.txt

@@ -26,6 +26,9 @@ gitpython~=3.1.43       # Latest stable
 # Security/Auth
 python-jose[cryptography]~=3.3.0  # JWT handling
 
+# Cloud storage
+google-cloud-storage~=2.18.0  # GCS offsite backups
+
 # Testing
 pytest~=8.3.4              # Test framework
 pytest-asyncio~=0.25.2     # Async test support

src/config.py

@@ -57,6 +57,7 @@ class Settings(BaseSettings):
     gitea_url: str = Field(default="http://gitea:3000", alias="GITEA_URL")
     gitea_user: str = Field(default="library", alias="GITEA_USER")
     gitea_password: str = Field(default="", alias="GITEA_PASSWORD")
+    gitea_token: str = Field(default="", alias="GITEA_TOKEN")  # API token with write:repository scope
     gitea_ssh_host: str = Field(default="gitea", alias="GITEA_SSH_HOST")
     gitea_ssh_port: int = Field(default=22, alias="GITEA_SSH_PORT")
 
@@ -65,6 +66,9 @@ class Settings(BaseSettings):
     backup_retention_weekly: int = Field(default=4, alias="BACKUP_RETENTION_WEEKLY")
     backup_retention_monthly: int = Field(default=12, alias="BACKUP_RETENTION_MONTHLY")
 
+    # Google Cloud Storage
+    gcs_credentials_file: str = Field(default="", alias="GCS_CREDENTIALS_FILE")
+
     # Documentation Mirroring
     docs_mirror_path: str = Field(default="/docs-mirror", alias="DOCS_MIRROR_PATH")
     docs_check_interval: int = Field(default=21600, alias="DOCS_CHECK_INTERVAL")  # 6 hours

src/executors/gcs_backup_executor.py

@@ -0,0 +1,199 @@
+"""
+Google Cloud Storage Backup Executor
+Backs up data to GCS buckets. Supports multiple modes:
+- git_bundle: Creates a git bundle from a bare repo and uploads it
+"""
+import asyncio
+import logging
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+from google.cloud import storage
+
+from src.config import Settings
+
+logger = logging.getLogger(__name__)
+
+
+async def execute(config: dict, settings: Settings) -> str:
+    """
+    Execute GCS backup task.
+
+    Config schema:
+        {
+            "mode": "git_bundle",
+            "bucket": "bucket-name",
+            "prefix": "gitea/settled-reach",
+            "credentials_path": "/secrets/gcs-sa-key.json",
+            "repo_path": "/data/docker-data/gitea/data/git/repositories/user/repo.git",
+            "retention_count": 7,
+            "dry_run": false
+        }
+
+    Args:
+        config: Backup configuration
+        settings: Global scheduler settings
+
+    Returns:
+        Summary of backup operation
+
+    Raises:
+        Exception: On backup failure
+    """
+    mode = config.get('mode')
+    if not mode:
+        raise ValueError("Missing required config field: mode")
+
+    if mode == 'git_bundle':
+        return await _mode_git_bundle(config, settings)
+    else:
+        raise ValueError(f"Unknown backup mode: {mode}")
+
+
+async def _mode_git_bundle(config: dict, settings: Settings) -> str:
+    """Create a git bundle from a bare repo and upload to GCS."""
+    bucket_name = config.get('bucket')
+    prefix = config.get('prefix', '').strip('/')
+    credentials_path = config.get('credentials_path', settings.gcs_credentials_file)
+    repo_path = Path(config.get('repo_path', ''))
+    retention_count = config.get('retention_count', 7)
+    dry_run = config.get('dry_run', False)
+
+    # Validate
+    if not bucket_name:
+        raise ValueError("Missing required config field: bucket")
+    if not repo_path or not str(repo_path).strip():
+        raise ValueError("Missing required config field: repo_path")
+    if not repo_path.exists():
+        raise ValueError(f"Repository path does not exist: {repo_path}")
+    if not (repo_path / 'HEAD').exists():
+        raise ValueError(f"Not a valid git repository: {repo_path}")
+
+    repo_name = repo_path.name.removesuffix('.git')
+    timestamp = datetime.now().strftime('%Y%m%d-%H%M%S')
+    bundle_filename = f"{repo_name}-{timestamp}.bundle"
+    bundle_path = Path(f"/tmp/{bundle_filename}")
+
+    logger.info(f"Starting GCS backup: mode=git_bundle, repo={repo_name}, dry_run={dry_run}")
+
+    try:
+        # Step 1: Create git bundle
+        logger.info(f"Creating git bundle from {repo_path}")
+        start = time.monotonic()
+
+        await _run_command([
+            'git', 'bundle', 'create',
+            str(bundle_path),
+            '--all'
+        ], cwd=repo_path)
+
+        bundle_duration = time.monotonic() - start
+
+        if not bundle_path.exists():
+            raise Exception("Git bundle was not created")
+
+        bundle_size = bundle_path.stat().st_size
+        bundle_size_mb = bundle_size / (1024 * 1024)
+        logger.info(f"Bundle created: {bundle_filename} ({bundle_size_mb:.1f} MB) in {bundle_duration:.1f}s")
+
+        # Step 2: Verify bundle
+        await _run_command([
+            'git', 'bundle', 'verify',
+            str(bundle_path)
+        ], cwd=repo_path)
+        logger.info("Bundle verified OK")
+
+        if dry_run:
+            return (
+                f"[DRY RUN] Would upload {bundle_filename} ({bundle_size_mb:.1f} MB) "
+                f"to gs://{bucket_name}/{prefix}/{bundle_filename}"
+            )
+
+        # Step 3: Upload to GCS
+        gcs_path = f"{prefix}/{bundle_filename}" if prefix else bundle_filename
+        logger.info(f"Uploading to gs://{bucket_name}/{gcs_path}")
+        start = time.monotonic()
+
+        client = storage.Client.from_service_account_json(credentials_path)
+        bucket = client.bucket(bucket_name)
+        blob = bucket.blob(gcs_path)
+        blob.upload_from_filename(str(bundle_path), timeout=3600)
+
+        upload_duration = time.monotonic() - start
+        logger.info(f"Upload complete in {upload_duration:.1f}s")
+
+        # Step 4: Retention cleanup in GCS
+        deleted_count = await _cleanup_gcs_retention(
+            client, bucket_name, prefix, repo_name, retention_count
+        )
+
+        return (
+            f"Backup completed: {bundle_filename} ({bundle_size_mb:.1f} MB). "
+            f"Bundle: {bundle_duration:.1f}s, Upload: {upload_duration:.1f}s. "
+            f"GCS: gs://{bucket_name}/{gcs_path}. "
+            f"Retention: {deleted_count} old bundle(s) removed."
+        )
+
+    finally:
+        # Always clean up the local temp file
+        if bundle_path.exists():
+            bundle_path.unlink()
+            logger.debug(f"Cleaned up temp file: {bundle_path}")
+
+
+async def _cleanup_gcs_retention(
+    client: storage.Client,
+    bucket_name: str,
+    prefix: str,
+    repo_name: str,
+    retention_count: int
+) -> int:
+    """Delete old bundles from GCS, keeping only the most recent retention_count."""
+    if retention_count <= 0:
+        return 0
+
+    bucket = client.bucket(bucket_name)
+    blob_prefix = f"{prefix}/{repo_name}-" if prefix else f"{repo_name}-"
+
+    blobs = list(bucket.list_blobs(prefix=blob_prefix))
+    bundle_blobs = [b for b in blobs if b.name.endswith('.bundle')]
+
+    if len(bundle_blobs) <= retention_count:
+        logger.info(f"Retention OK: {len(bundle_blobs)} bundles (limit: {retention_count})")
+        return 0
+
+    # Sort by name (timestamp in name ensures chronological order)
+    bundle_blobs.sort(key=lambda b: b.name)
+    to_delete = bundle_blobs[:-retention_count]
+
+    for blob in to_delete:
+        logger.info(f"Deleting old bundle: {blob.name}")
+        blob.delete()
+
+    logger.info(f"Retention cleanup: deleted {len(to_delete)} old bundle(s)")
+    return len(to_delete)
+
+
+async def _run_command(
+    cmd: List[str],
+    cwd: Optional[Path] = None,
+) -> str:
+    """Run shell command asynchronously."""
+    logger.debug(f"Running: {' '.join(cmd)} (cwd: {cwd})")
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        cwd=cwd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE
+    )
+
+    stdout, stderr = await proc.communicate()
+
+    if proc.returncode != 0:
+        error_msg = stderr.decode() if stderr else "Unknown error"
+        raise Exception(f"Command failed: {' '.join(cmd)}\n{error_msg}")
+
+    return stdout.decode()

src/executors/gitea_release_cleanup_executor.py

@@ -0,0 +1,211 @@
+"""
+Gitea Release Cleanup Executor
+
+Cleans up old releases across all Gitea repositories, keeping only the most recent
+releases per repository. Designed to run before Watchtower to prevent accumulation
+of old container image tags.
+
+Config schema:
+    {
+        "keep_count": 5,           # Number of releases to keep per repo (default: 5)
+        "exclude_repos": [],       # Repository names to skip (default: [])
+        "dry_run": false           # If true, only log what would be deleted (default: false)
+    }
+
+Example config:
+    {
+        "keep_count": 5,
+        "exclude_repos": ["important-repo", "legacy-app"],
+        "dry_run": false
+    }
+
+Required settings:
+    - GITEA_URL: Base URL of Gitea instance
+    - GITEA_TOKEN: API token with write:repository scope
+"""
+import logging
+from typing import Any
+
+import httpx
+
+from src.config import Settings
+
+logger = logging.getLogger(__name__)
+
+# Default configuration values
+DEFAULT_KEEP_COUNT = 5
+DEFAULT_TIMEOUT = 60
+API_BASE = "/api/v1"
+
+
+async def execute(config: dict, settings: Settings) -> str:
+    """
+    Clean up old releases across all accessible Gitea repositories.
+
+    Args:
+        config: Task configuration (see module docstring)
+        settings: Global scheduler settings
+
+    Returns:
+        Summary of cleanup actions taken
+
+    Raises:
+        ValueError: On configuration error
+        Exception: On API call failure
+    """
+    # Validate settings
+    if not settings.gitea_url:
+        raise ValueError("GITEA_URL not configured")
+    if not settings.gitea_token:
+        raise ValueError("GITEA_TOKEN not configured - required for release deletion")
+
+    # Parse configuration
+    keep_count = config.get("keep_count", DEFAULT_KEEP_COUNT)
+    exclude_repos = config.get("exclude_repos", [])
+    dry_run = config.get("dry_run", False)
+
+    if keep_count < 1:
+        raise ValueError(f"keep_count must be at least 1, got {keep_count}")
+
+    base_url = settings.gitea_url.rstrip("/")
+    headers = {"Authorization": f"token {settings.gitea_token}"}
+
+    stats = {
+        "repos_scanned": 0,
+        "repos_with_releases": 0,
+        "releases_deleted": 0,
+        "releases_skipped": 0,
+        "errors": [],
+    }
+
+    mode = "DRY RUN" if dry_run else "LIVE"
+    logger.info(f"Starting Gitea release cleanup ({mode}): keeping {keep_count} releases per repo")
+
+    async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT) as client:
+        # Fetch all repositories
+        repos = await _fetch_user_repos(client, base_url, headers)
+        logger.info(f"Found {len(repos)} repositories")
+
+        for repo in repos:
+            owner = repo["owner"]["login"]
+            name = repo["name"]
+            full_name = f"{owner}/{name}"
+
+            # Check exclusion list
+            if name in exclude_repos or full_name in exclude_repos:
+                logger.debug(f"Skipping excluded repo: {full_name}")
+                continue
+
+            stats["repos_scanned"] += 1
+
+            try:
+                # Fetch releases for this repo
+                releases = await _fetch_releases(client, base_url, headers, owner, name)
+
+                if not releases:
+                    continue
+
+                stats["repos_with_releases"] += 1
+
+                # Determine which releases to delete (beyond keep_count)
+                to_delete = releases[keep_count:]
+
+                if not to_delete:
+                    logger.debug(f"{full_name}: {len(releases)} releases, nothing to delete")
+                    continue
+
+                logger.info(f"{full_name}: {len(releases)} releases, deleting {len(to_delete)}")
+
+                # Delete old releases
+                for release in to_delete:
+                    release_id = release["id"]
+                    tag_name = release["tag_name"]
+
+                    if dry_run:
+                        logger.info(f"  [DRY RUN] Would delete: {tag_name} (id={release_id})")
+                        stats["releases_skipped"] += 1
+                    else:
+                        try:
+                            await _delete_release(client, base_url, headers, owner, name, release_id)
+                            logger.info(f"  Deleted: {tag_name}")
+                            stats["releases_deleted"] += 1
+                        except Exception as e:
+                            error_msg = f"{full_name}/{tag_name}: {e}"
+                            logger.warning(f"  Failed to delete {tag_name}: {e}")
+                            stats["errors"].append(error_msg)
+
+            except Exception as e:
+                error_msg = f"{full_name}: {e}"
+                logger.error(f"Error processing {full_name}: {e}")
+                stats["errors"].append(error_msg)
+
+    # Build summary
+    summary = _build_summary(stats, dry_run)
+    logger.info(f"Cleanup complete: {summary}")
+
+    return summary
+
+
+async def _fetch_user_repos(
+    client: httpx.AsyncClient,
+    base_url: str,
+    headers: dict,
+) -> list[dict[str, Any]]:
+    """Fetch all repositories accessible to the authenticated user."""
+    url = f"{base_url}{API_BASE}/user/repos"
+    params = {"limit": 100}
+
+    response = await client.get(url, headers=headers, params=params)
+    response.raise_for_status()
+
+    return response.json()
+
+
+async def _fetch_releases(
+    client: httpx.AsyncClient,
+    base_url: str,
+    headers: dict,
+    owner: str,
+    repo: str,
+) -> list[dict[str, Any]]:
+    """Fetch releases for a repository, sorted newest first (Gitea default)."""
+    url = f"{base_url}{API_BASE}/repos/{owner}/{repo}/releases"
+    params = {"limit": 100}
+
+    response = await client.get(url, headers=headers, params=params)
+    response.raise_for_status()
+
+    return response.json()
+
+
+async def _delete_release(
+    client: httpx.AsyncClient,
+    base_url: str,
+    headers: dict,
+    owner: str,
+    repo: str,
+    release_id: int,
+) -> None:
+    """Delete a specific release."""
+    url = f"{base_url}{API_BASE}/repos/{owner}/{repo}/releases/{release_id}"
+
+    response = await client.delete(url, headers=headers)
+    response.raise_for_status()
+
+
+def _build_summary(stats: dict, dry_run: bool) -> str:
+    """Build a human-readable summary of the cleanup operation."""
+    parts = [
+        f"Scanned {stats['repos_scanned']} repos",
+        f"{stats['repos_with_releases']} with releases",
+    ]
+
+    if dry_run:
+        parts.append(f"{stats['releases_skipped']} releases would be deleted")
+    else:
+        parts.append(f"{stats['releases_deleted']} releases deleted")
+
+    if stats["errors"]:
+        parts.append(f"{len(stats['errors'])} errors")
+
+    return ", ".join(parts)