fix(health): bound and parallelise the five dependency probes
check_service_health() ran neo4j, qdrant, wikijs, searxng and ollama serially with await, and only ollama's client carried its own timeout. Neo4j (connection_timeout=30.0) and Qdrant (timeout=30.0) fell back to driver defaults far past the container healthcheck's 10s timeout, so a hung (not failing) dependency blocked the whole chain and flipped the container unhealthy for a reason unrelated to its own liveness. Each probe now runs under asyncio.wait_for bounded at 2s — chosen against the 10s healthcheck timeout so five concurrent bounded probes cannot approach it even if all five hang — and all five run concurrently under asyncio.gather(return_exceptions=True), so one probe timing out or raising cannot block or cancel the others. Gating (neo4j+qdrant only), the unconditional 200 response, ollama's existing 5.0s client-level timeout, and the paperless/system_settings/ scheduler probes are unchanged.
This commit is contained in:
@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- `/health`'s five dependency probes (neo4j, qdrant, wikijs, searxng, ollama)
|
||||
now run concurrently under `asyncio.gather`, each bounded at 2s via
|
||||
`asyncio.wait_for`, instead of serially with no bound on neo4j or qdrant.
|
||||
A hung (not failing) dependency previously blocked the whole chain past the
|
||||
container healthcheck's 10s timeout, marking the container unhealthy for a
|
||||
reason unrelated to its own liveness. A probe that times out now reports
|
||||
that service unhealthy, the same as one that raises; the others are
|
||||
unaffected.
|
||||
|
||||
## [1.9.1] - 2026-08-08
|
||||
|
||||
### Fixed
|
||||
|
||||
+116
-46
@@ -8,6 +8,7 @@ Provides FastAPI dependencies for service clients with:
|
||||
- Type aliases for clean endpoint signatures
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
from fastapi import Depends, HTTPException, Query
|
||||
@@ -565,10 +566,100 @@ async def shutdown_clients():
|
||||
logger.info("Service clients shutdown complete")
|
||||
|
||||
|
||||
# Per-probe timeout for the concurrent health checks below, in seconds.
|
||||
#
|
||||
# Bounded well under the container healthcheck's 10s timeout (see Dockerfile)
|
||||
# so that five probes run concurrently under asyncio.gather cannot approach
|
||||
# it even if all five hang. A probe that has no timeout of its own (neo4j,
|
||||
# qdrant) otherwise falls back to its driver's default — 30s for both — which
|
||||
# is what let a hung dependency (not a failing one) flip the container
|
||||
# unhealthy.
|
||||
HEALTH_PROBE_TIMEOUT = 2.0
|
||||
|
||||
|
||||
async def _probe_neo4j() -> bool:
|
||||
"""Neo4j connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
neo4j = get_neo4j_client()
|
||||
# Simple query to check connectivity
|
||||
await neo4j.execute_query("RETURN 1 as test", {})
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Neo4j health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_qdrant() -> bool:
|
||||
"""Qdrant connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
qdrant = get_qdrant_client()
|
||||
# Check if we can list collections
|
||||
await qdrant.client.get_collections()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Qdrant health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_wikijs() -> bool:
|
||||
"""Wiki.js connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
wikijs = get_wikijs_client()
|
||||
# Try a simple query (list pages with limit 1)
|
||||
await wikijs.list_pages(limit=1)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Wiki.js health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_searxng() -> bool:
|
||||
"""SearXNG connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
searxng = get_searxng_client()
|
||||
# Just check if service is up (no actual search)
|
||||
return await searxng.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"SearXNG health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_ollama() -> bool:
|
||||
"""Ollama connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
ollama = get_ollama_client()
|
||||
return await ollama.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"Ollama health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _bounded_probe(coro) -> bool:
|
||||
"""
|
||||
Run a probe coroutine bounded by HEALTH_PROBE_TIMEOUT.
|
||||
|
||||
A probe that times out is reported unhealthy, the same as one that
|
||||
raises. Wrapping happens here rather than in each _probe_* function so
|
||||
the bound applies uniformly regardless of whether the underlying client
|
||||
has its own (looser, or absent) timeout.
|
||||
"""
|
||||
try:
|
||||
return await asyncio.wait_for(coro, timeout=HEALTH_PROBE_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"Health probe timed out after {HEALTH_PROBE_TIMEOUT}s")
|
||||
return False
|
||||
|
||||
|
||||
async def check_service_health() -> dict:
|
||||
"""
|
||||
Check health of all service clients.
|
||||
|
||||
The five core probes (neo4j, qdrant, wikijs, searxng, ollama) run
|
||||
concurrently, each bounded at HEALTH_PROBE_TIMEOUT, so a single hung
|
||||
dependency cannot block the others or push the endpoint past the
|
||||
container healthcheck's timeout. return_exceptions=True on the gather
|
||||
means one probe raising cannot cancel its siblings.
|
||||
|
||||
Returns:
|
||||
Dictionary with health status of each service:
|
||||
{
|
||||
@@ -586,53 +677,32 @@ async def check_service_health() -> dict:
|
||||
"""
|
||||
health = {}
|
||||
|
||||
# Neo4j
|
||||
try:
|
||||
neo4j = get_neo4j_client()
|
||||
# Simple query to check connectivity
|
||||
await neo4j.execute_query("RETURN 1 as test", {})
|
||||
health["neo4j"] = True
|
||||
except Exception as e:
|
||||
logger.error(f"Neo4j health check failed: {e}")
|
||||
health["neo4j"] = False
|
||||
neo4j_result, qdrant_result, wikijs_result, searxng_result, ollama_result = (
|
||||
await asyncio.gather(
|
||||
_bounded_probe(_probe_neo4j()),
|
||||
_bounded_probe(_probe_qdrant()),
|
||||
_bounded_probe(_probe_wikijs()),
|
||||
_bounded_probe(_probe_searxng()),
|
||||
_bounded_probe(_probe_ollama()),
|
||||
return_exceptions=True,
|
||||
)
|
||||
)
|
||||
|
||||
# Qdrant
|
||||
try:
|
||||
qdrant = get_qdrant_client()
|
||||
# Check if we can list collections
|
||||
await qdrant.client.get_collections()
|
||||
health["qdrant"] = True
|
||||
except Exception as e:
|
||||
logger.error(f"Qdrant health check failed: {e}")
|
||||
health["qdrant"] = False
|
||||
|
||||
# Wiki.js
|
||||
try:
|
||||
wikijs = get_wikijs_client()
|
||||
# Try a simple query (list pages with limit 1)
|
||||
await wikijs.list_pages(limit=1)
|
||||
health["wikijs"] = True
|
||||
except Exception as e:
|
||||
logger.error(f"Wiki.js health check failed: {e}")
|
||||
health["wikijs"] = False
|
||||
|
||||
# SearXNG
|
||||
try:
|
||||
searxng = get_searxng_client()
|
||||
# Just check if service is up (no actual search)
|
||||
health["searxng"] = await searxng.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"SearXNG health check failed: {e}")
|
||||
health["searxng"] = False
|
||||
|
||||
# Ollama
|
||||
try:
|
||||
ollama = get_ollama_client()
|
||||
is_healthy = await ollama.health_check()
|
||||
health["ollama"] = is_healthy
|
||||
except Exception as e:
|
||||
logger.error(f"Ollama health check failed: {e}")
|
||||
health["ollama"] = False
|
||||
# _bounded_probe already catches everything from its own probe, but
|
||||
# return_exceptions=True also guards against a bug in _bounded_probe
|
||||
# itself surfacing as an unhandled exception here.
|
||||
for name, result in (
|
||||
("neo4j", neo4j_result),
|
||||
("qdrant", qdrant_result),
|
||||
("wikijs", wikijs_result),
|
||||
("searxng", searxng_result),
|
||||
("ollama", ollama_result),
|
||||
):
|
||||
if isinstance(result, BaseException):
|
||||
logger.error(f"{name} health check raised unexpectedly: {result}")
|
||||
health[name] = False
|
||||
else:
|
||||
health[name] = result
|
||||
|
||||
# Paperless-ngx
|
||||
settings = get_settings()
|
||||
|
||||
Reference in New Issue
Block a user