feat: add job cleanup loop, Scheduler task definitions, and registrar

- job_cleanup_loop (src/jobs/job_manager.py): hourly in-process pass over
  JobManager.cleanup_expired_jobs, started at app startup and cancelled
  at shutdown; Redis job payloads auto-expire but set memberships do not.
- docs/scheduler-tasks.md: the four production Scheduler task payloads
  for the deploy checklist - nightly integrity check 04:30, weekly
  quality report Sunday 03:00 (day_of_week=6, 0=Monday), daily Paperless
  orphan-cleanup 05:00 hitting the existing
  /maintenance/cleanup/paperless?user=jpmschweitzer&dry_run=false
  endpoint, and disabling test_example_task - with exact HTTP bodies
  (explicit user=jpmschweitzer, Authorization: Bearer ${LIBRARY_API_KEY}
  placeholder).
- scripts/register_scheduler_tasks.py: reads SCHEDULER_URL from env,
  DRY-RUN BY DEFAULT (prints the exact payloads, provably contacts
  nothing), --execute gated and requiring LIBRARY_API_KEY to fill the
  placeholder. NOT executed - definitions delivered for the deploy
  checklist only.

9 new offline tests (loop passes/error-resilience/cancellation, payload
schedules, explicit production user, placeholder, dry-run default).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
This commit is contained in:
2026-07-14 12:30:27 +02:00
co-authored by Claude Fable 5
parent 51f9ce08ec
commit 0b346d3a57
6 changed files with 581 additions and 1 deletions
+36
View File
@@ -9,6 +9,7 @@ Provides background job management with:
- User-scoped job queries
"""
import asyncio
import redis.asyncio as redis
import json
import uuid
@@ -424,3 +425,38 @@ class JobManager:
stats[status] += 1
return stats
async def job_cleanup_loop(
job_manager: JobManager,
interval_seconds: float = 3600,
max_iterations: Optional[int] = None
) -> int:
"""
Periodically clean up expired job-set memberships.
Redis auto-expires the job payloads (24h TTL) but set memberships
(library:active_jobs, library:user_jobs:{user}) need manual cleanup.
Started as an in-process background task at application startup.
Args:
job_manager: JobManager whose cleanup_expired_jobs is invoked
interval_seconds: Sleep between cleanup passes (default hourly)
max_iterations: Stop after N passes (None = run forever; used by tests)
Returns:
Number of completed cleanup passes (only reachable with max_iterations)
"""
iterations = 0
while max_iterations is None or iterations < max_iterations:
try:
await asyncio.sleep(interval_seconds)
await job_manager.cleanup_expired_jobs()
except asyncio.CancelledError:
logger.info("Job cleanup loop cancelled")
raise
except Exception as e:
# Never let a transient Redis error kill the loop
logger.error(f"Job cleanup pass failed: {e}")
iterations += 1
return iterations
+20 -1
View File
@@ -598,7 +598,10 @@ async def check_duplicates(
@app.on_event("startup")
async def startup_event():
"""Initialize connections and resources on startup."""
from src.core.dependencies import startup_clients
import asyncio
from src.core.dependencies import startup_clients, get_job_manager
from src.jobs.job_manager import job_cleanup_loop
from src.services.wiki_change_listener import WikiChangeListener
settings = get_settings()
@@ -613,6 +616,13 @@ async def startup_event():
# Initialize all service clients
await startup_clients()
# Hourly in-process cleanup of expired Redis job-set memberships
# (job payloads auto-expire via TTL; set memberships do not)
app.state.job_cleanup_task = asyncio.create_task(
job_cleanup_loop(get_job_manager(), interval_seconds=3600)
)
logger.info("Job cleanup loop started (hourly)")
# Start Wiki.js change listener (PostgreSQL NOTIFY/LISTEN)
# This enables automatic processing of user-edited pages
try:
@@ -633,6 +643,15 @@ async def shutdown_event():
logger.info("Shutting down Library Desk API")
# Stop the job cleanup loop
if hasattr(app.state, "job_cleanup_task"):
app.state.job_cleanup_task.cancel()
try:
await app.state.job_cleanup_task
except Exception:
pass
logger.info("Job cleanup loop stopped")
# Stop Wiki.js change listener if running
if hasattr(app.state, "wiki_listener"):
try: