diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ac184c..4e689db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Added +- Backup executors report their own outcome to the homelab health record — one row in + `check_history` per run, success or failure. Replaces a monitor that inferred backup health + from file age and could not tell a failed backup from one that had not run yet. + +### Notes +- Requires `GRANT INSERT ON check_history TO scheduler_user` in the `sysmon` database. Without + it the report is refused, logged, and skipped; the backup itself is unaffected. + ## [1.4.0] - 2026-08-08 ### Added @@ -271,6 +280,15 @@ TOTAL 80% 🎯 ## [Unreleased] +### Added +- Backup executors report their own outcome to the homelab health record — one row in + `check_history` per run, success or failure. Replaces a monitor that inferred backup health + from file age and could not tell a failed backup from one that had not run yet. + +### Notes +- Requires `GRANT INSERT ON check_history TO scheduler_user` in the `sysmon` database. Without + it the report is refused, logged, and skipped; the backup itself is unaffected. + ### Planned - Redis integration for distributed locking - Webhook notifications for task completion diff --git a/src/executors/config_backup_executor.py b/src/executors/config_backup_executor.py index ef61728..202c768 100644 --- a/src/executors/config_backup_executor.py +++ b/src/executors/config_backup_executor.py @@ -12,11 +12,12 @@ from pathlib import Path from typing import List, Dict, Any from src.config import Settings +from src.executors import health_report logger = logging.getLogger(__name__) -async def execute(config: dict, settings: Settings) -> str: +async def _run(config: dict, settings: Settings) -> str: """ Execute config backup task. @@ -157,3 +158,33 @@ async def cleanup_old_backups(backup_dir: Path, retention_days: int): logger.info(f"Removed {removed_count} old backups, freed {removed_size_mb:.2f}MB") else: logger.info("No old backups to remove") + + +async def execute(config: dict, settings: Settings) -> str: + """Run the backup and report its own outcome to check_history (D-33, T-69). + + The report wraps the work rather than living inside it, so the failure path + cannot be forgotten: an exception is reported as critical and then re-raised, + leaving the task's own status untouched. Reporting only success would + reproduce exactly the blind spot this replaces — a monitor that cannot tell + a failed backup from one that has not run. + """ + try: + output = await _run(config, settings) + except Exception as exc: + health_report.report( + settings, + domain="backup", + status=health_report.CRITICAL, + source="scheduler/config_backup_executor", + metrics={"job": "scheduler/config_backup_executor", "error": str(exc)[:400]}, + ) + raise + health_report.report( + settings, + domain="backup", + status=health_report.OK, + source="scheduler/config_backup_executor", + metrics={"job": "scheduler/config_backup_executor", "summary": output[:400]}, + ) + return output diff --git a/src/executors/health_report.py b/src/executors/health_report.py new file mode 100644 index 0000000..2327d03 --- /dev/null +++ b/src/executors/health_report.py @@ -0,0 +1,127 @@ +"""Report a task's own outcome to the homelab's central health record. + +Why a process reports itself, rather than a monitor inferring it: + +The sysmon `backup` domain used to poll the mtime of the newest file in the +backup directory, hourly, against a 48-hour threshold — for a job that runs once +a day. Forty-seven of every forty-eight runs could not produce a new answer, and +worse, a file-age poll cannot distinguish "the backup failed" from "the backup +has not run yet". If tonight's job dies, yesterday's archive is 24 hours old and +still reads healthy, and keeps reading healthy until hour 48. A failure stayed +invisible for two days to the check whose only job was noticing it. + +This executor knows at 03:05. So it says so. + +Recorded as D-33 in the workspace vault: `check_history` is the central health +record and any self-maintained service may push a row describing its own +outcome. sysmon polls only the things that cannot report themselves. + +Three consequences that are load-bearing here: + +- `source` names the producer, because the table now has several writers and a + row must say which one wrote it. +- `domain` is a shared namespace. Two producers claiming one name would + interleave silently. +- **A reporting failure must never fail the task.** Backing up successfully and + failing to mention it is strictly better than the reverse. Everything here is + caught and logged, which means the absence of rows is the only symptom a + broken reporter produces — so check for rows, not for errors. +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +import psycopg2 + +logger = logging.getLogger(__name__) + +# The database holding check_history. Not the scheduler's own database — this is +# a cross-service write into the health record, and it is deliberate (D-33). +HEALTH_DB = "sysmon" + +OK = "ok" +WARNING = "warning" +CRITICAL = "critical" + + +def report( + settings: Any, + domain: str, + status: str, + source: str, + metrics: Optional[Dict[str, Any]] = None, +) -> bool: + """Write one row to check_history. Returns whether it landed. + + Never raises. A caller that lets this failure surface would turn a + successful backup into a failed task, which inverts the point. + """ + metrics = metrics or {} + now = datetime.now(timezone.utc) + result = { + # The envelope the table has carried since the shell era. A reader of a + # year of history should not have to know which producer wrote a row in + # order to parse it. + "timestamp": now.isoformat(), + "source": source, + "domain": domain, + "status": status, + "metrics": metrics, + } + + try: + conn = psycopg2.connect( + host=settings.postgres_host, + port=settings.postgres_port, + database=HEALTH_DB, + user=settings.postgres_user, + password=settings.postgres_password, + connect_timeout=10, + ) + except Exception as exc: # noqa: BLE001 - reporting must not raise + logger.warning("health report for %s could not connect to %s: %s", domain, HEALTH_DB, exc) + return False + + try: + with conn: + with conn.cursor() as cur: + # Unqualified table name, resolved through the search_path of the + # sysmon database. Qualifying it as sysmon.check_history looks + # more careful and is wrong — that schema does not exist. + cur.execute( + "INSERT INTO check_history (host, domain, status, ts, result) " + "VALUES (%s, %s, %s, %s, %s)", + (_host(), domain, status, now, json.dumps(result)), + ) + logger.info("health report: %s=%s recorded", domain, status) + return True + except psycopg2.errors.InsufficientPrivilege: + # Named separately because it is the expected first failure and the fix + # is a one-line grant, not a code change: + # GRANT INSERT ON check_history TO ; + logger.warning( + "health report for %s refused: the scheduler's database user lacks INSERT on " + "check_history. The task itself succeeded; only the report was lost.", + domain, + ) + return False + except Exception as exc: # noqa: BLE001 - reporting must not raise + logger.warning("health report for %s failed: %s", domain, exc) + return False + finally: + conn.close() + + +def _host() -> str: + """The host a row is attributed to. + + Every Redis key and check_history row is scoped by host so a second machine + reporting into the same store stays distinguishable. The scheduler runs in a + container, whose hostname is a container id — useless as an attribution — so + the physical host is named explicitly. + """ + import os + + return os.environ.get("SYSMON_HOST", "tower-of-joy") diff --git a/src/executors/portainer_backup_executor.py b/src/executors/portainer_backup_executor.py index f9f0f83..3f21032 100644 --- a/src/executors/portainer_backup_executor.py +++ b/src/executors/portainer_backup_executor.py @@ -37,6 +37,7 @@ from pathlib import Path import httpx from src.config import Settings +from src.executors import health_report logger = logging.getLogger(__name__) @@ -72,7 +73,7 @@ def _prune(output_dir: Path, retention_days: int) -> int: return removed -async def execute(config: dict, settings: Settings) -> str: +async def _run(config: dict, settings: Settings) -> str: url = _substitute_env(config.get("url", "")).rstrip("/") api_key = _substitute_env(config.get("api_key", "")) output_dir = Path(config.get("output_dir", "/backups/portainer")) @@ -138,3 +139,33 @@ async def execute(config: dict, settings: Settings) -> str: ) logger.info(summary) return summary + + +async def execute(config: dict, settings: Settings) -> str: + """Run the backup and report its own outcome to check_history (D-33, T-69). + + The report wraps the work rather than living inside it, so the failure path + cannot be forgotten: an exception is reported as critical and then re-raised, + leaving the task's own status untouched. Reporting only success would + reproduce exactly the blind spot this replaces — a monitor that cannot tell + a failed backup from one that has not run. + """ + try: + output = await _run(config, settings) + except Exception as exc: + health_report.report( + settings, + domain="backup", + status=health_report.CRITICAL, + source="scheduler/portainer_backup_executor", + metrics={"job": "scheduler/portainer_backup_executor", "error": str(exc)[:400]}, + ) + raise + health_report.report( + settings, + domain="backup", + status=health_report.OK, + source="scheduler/portainer_backup_executor", + metrics={"job": "scheduler/portainer_backup_executor", "summary": output[:400]}, + ) + return output