feat: add job cleanup loop, Scheduler task definitions, and registrar
- job_cleanup_loop (src/jobs/job_manager.py): hourly in-process pass over
JobManager.cleanup_expired_jobs, started at app startup and cancelled
at shutdown; Redis job payloads auto-expire but set memberships do not.
- docs/scheduler-tasks.md: the four production Scheduler task payloads
for the deploy checklist - nightly integrity check 04:30, weekly
quality report Sunday 03:00 (day_of_week=6, 0=Monday), daily Paperless
orphan-cleanup 05:00 hitting the existing
/maintenance/cleanup/paperless?user=jpmschweitzer&dry_run=false
endpoint, and disabling test_example_task - with exact HTTP bodies
(explicit user=jpmschweitzer, Authorization: Bearer ${LIBRARY_API_KEY}
placeholder).
- scripts/register_scheduler_tasks.py: reads SCHEDULER_URL from env,
DRY-RUN BY DEFAULT (prints the exact payloads, provably contacts
nothing), --execute gated and requiring LIBRARY_API_KEY to fill the
placeholder. NOT executed - definitions delivered for the deploy
checklist only.
9 new offline tests (loop passes/error-resilience/cancellation, payload
schedules, explicit production user, placeholder, dry-run default).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
This commit is contained in:
@@ -9,6 +9,7 @@ Provides background job management with:
|
||||
- User-scoped job queries
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import redis.asyncio as redis
|
||||
import json
|
||||
import uuid
|
||||
@@ -424,3 +425,38 @@ class JobManager:
|
||||
stats[status] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
async def job_cleanup_loop(
|
||||
job_manager: JobManager,
|
||||
interval_seconds: float = 3600,
|
||||
max_iterations: Optional[int] = None
|
||||
) -> int:
|
||||
"""
|
||||
Periodically clean up expired job-set memberships.
|
||||
|
||||
Redis auto-expires the job payloads (24h TTL) but set memberships
|
||||
(library:active_jobs, library:user_jobs:{user}) need manual cleanup.
|
||||
Started as an in-process background task at application startup.
|
||||
|
||||
Args:
|
||||
job_manager: JobManager whose cleanup_expired_jobs is invoked
|
||||
interval_seconds: Sleep between cleanup passes (default hourly)
|
||||
max_iterations: Stop after N passes (None = run forever; used by tests)
|
||||
|
||||
Returns:
|
||||
Number of completed cleanup passes (only reachable with max_iterations)
|
||||
"""
|
||||
iterations = 0
|
||||
while max_iterations is None or iterations < max_iterations:
|
||||
try:
|
||||
await asyncio.sleep(interval_seconds)
|
||||
await job_manager.cleanup_expired_jobs()
|
||||
except asyncio.CancelledError:
|
||||
logger.info("Job cleanup loop cancelled")
|
||||
raise
|
||||
except Exception as e:
|
||||
# Never let a transient Redis error kill the loop
|
||||
logger.error(f"Job cleanup pass failed: {e}")
|
||||
iterations += 1
|
||||
return iterations
|
||||
|
||||
+20
-1
@@ -598,7 +598,10 @@ async def check_duplicates(
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
"""Initialize connections and resources on startup."""
|
||||
from src.core.dependencies import startup_clients
|
||||
import asyncio
|
||||
|
||||
from src.core.dependencies import startup_clients, get_job_manager
|
||||
from src.jobs.job_manager import job_cleanup_loop
|
||||
from src.services.wiki_change_listener import WikiChangeListener
|
||||
|
||||
settings = get_settings()
|
||||
@@ -613,6 +616,13 @@ async def startup_event():
|
||||
# Initialize all service clients
|
||||
await startup_clients()
|
||||
|
||||
# Hourly in-process cleanup of expired Redis job-set memberships
|
||||
# (job payloads auto-expire via TTL; set memberships do not)
|
||||
app.state.job_cleanup_task = asyncio.create_task(
|
||||
job_cleanup_loop(get_job_manager(), interval_seconds=3600)
|
||||
)
|
||||
logger.info("Job cleanup loop started (hourly)")
|
||||
|
||||
# Start Wiki.js change listener (PostgreSQL NOTIFY/LISTEN)
|
||||
# This enables automatic processing of user-edited pages
|
||||
try:
|
||||
@@ -633,6 +643,15 @@ async def shutdown_event():
|
||||
|
||||
logger.info("Shutting down Library Desk API")
|
||||
|
||||
# Stop the job cleanup loop
|
||||
if hasattr(app.state, "job_cleanup_task"):
|
||||
app.state.job_cleanup_task.cancel()
|
||||
try:
|
||||
await app.state.job_cleanup_task
|
||||
except Exception:
|
||||
pass
|
||||
logger.info("Job cleanup loop stopped")
|
||||
|
||||
# Stop Wiki.js change listener if running
|
||||
if hasattr(app.state, "wiki_listener"):
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user