feat: add external API providers and central settings database
- Add central settings database client (system_settings on postgres-shared) - User-scoped settings with global fallback - API config storage with enabled/disabled toggle - Per-source category filtering for news - Add modular external API providers in src/apis/: - OpenMeteoProvider: weather with geocoding (free, no key) - NOSProvider: Dutch news RSS feeds - BBCProvider: English news RSS feeds - AggregatedNewsProvider: merges sources with category filtering - AlphaVantageProvider: financial quotes (API key from settings DB) - Add provider dependencies and lifecycle management - Add requirements-dev.txt with pip-audit for security auditing - Add MEMORY_REMEMBER_PLAN.md documenting volatile/document memory architecture 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -23,6 +23,12 @@ from src.clients.searxng_client import SearXNGClient
|
||||
from src.clients.ollama_client import OllamaClient
|
||||
from src.clients.content_extractor import ContentExtractor
|
||||
from src.clients.paperless_client import PaperlessClient
|
||||
from src.clients.settings_client import SettingsClient
|
||||
from src.apis import (
|
||||
OpenMeteoProvider,
|
||||
AggregatedNewsProvider,
|
||||
AlphaVantageProvider,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -178,6 +184,131 @@ def get_paperless_client() -> PaperlessClient:
|
||||
return client
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings_client() -> SettingsClient:
|
||||
"""
|
||||
Get central settings database client singleton.
|
||||
|
||||
Returns:
|
||||
Initialized SettingsClient for Tatlock system_settings database
|
||||
|
||||
Note: Returns client with empty DSN if password not configured
|
||||
"""
|
||||
settings = get_settings()
|
||||
if not settings.system_settings_password:
|
||||
logger.warning("System settings password not configured - settings database disabled")
|
||||
client = SettingsClient(dsn=settings.system_settings_dsn)
|
||||
logger.debug(f"Created Settings client: {settings.system_settings_host}")
|
||||
return client
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# External API Providers
|
||||
# =============================================================================
|
||||
|
||||
@lru_cache
|
||||
def get_weather_provider() -> OpenMeteoProvider:
|
||||
"""
|
||||
Get Open-Meteo weather provider singleton.
|
||||
|
||||
Returns:
|
||||
Initialized OpenMeteoProvider with default timezone
|
||||
|
||||
Note: Timezone can be overridden per-request for user preferences
|
||||
"""
|
||||
provider = OpenMeteoProvider(timezone="Europe/Amsterdam")
|
||||
logger.debug("Created OpenMeteo weather provider")
|
||||
return provider
|
||||
|
||||
|
||||
# News provider requires sources from settings database
|
||||
_news_provider: AggregatedNewsProvider | None = None
|
||||
|
||||
|
||||
async def get_news_provider() -> AggregatedNewsProvider:
|
||||
"""
|
||||
Get aggregated news provider.
|
||||
|
||||
Returns:
|
||||
Initialized AggregatedNewsProvider with user-configured sources
|
||||
and per-source category filters.
|
||||
|
||||
Note: Configuration is fetched from system_settings database:
|
||||
- news.sources: list of enabled sources (default: ["nos", "bbc"])
|
||||
- api.{source}.categories: list of enabled categories per source
|
||||
"""
|
||||
global _news_provider
|
||||
if _news_provider is not None:
|
||||
return _news_provider
|
||||
|
||||
settings_client = get_settings_client()
|
||||
|
||||
# Get enabled sources
|
||||
sources = await settings_client.get("news.sources")
|
||||
if not sources or not isinstance(sources, list):
|
||||
sources = ["nos", "bbc"]
|
||||
logger.info(f"Using default news sources: {sources}")
|
||||
else:
|
||||
logger.info(f"Using configured news sources: {sources}")
|
||||
|
||||
# Filter out disabled sources and get category filters
|
||||
enabled_sources: list[str] = []
|
||||
category_filters: dict[str, list[str]] = {}
|
||||
|
||||
for source in sources:
|
||||
config = await settings_client.get_api_config(source)
|
||||
if config:
|
||||
# Check if source is disabled
|
||||
if config.get("enabled") is False:
|
||||
logger.info(f"News source '{source}' is disabled - skipping")
|
||||
continue
|
||||
# Get category filter if specified
|
||||
categories = config.get("categories", [])
|
||||
if categories:
|
||||
category_filters[source] = categories
|
||||
logger.debug(f"Source '{source}' categories: {categories}")
|
||||
enabled_sources.append(source)
|
||||
|
||||
if not enabled_sources:
|
||||
enabled_sources = ["nos", "bbc"]
|
||||
logger.warning("No enabled news sources - using defaults")
|
||||
|
||||
_news_provider = AggregatedNewsProvider(
|
||||
sources=enabled_sources,
|
||||
category_filters=category_filters
|
||||
)
|
||||
return _news_provider
|
||||
|
||||
|
||||
# AlphaVantage requires API key from settings database
|
||||
_alphavantage_provider: AlphaVantageProvider | None = None
|
||||
|
||||
|
||||
async def get_alphavantage_provider() -> AlphaVantageProvider | None:
|
||||
"""
|
||||
Get Alpha Vantage financial provider.
|
||||
|
||||
Returns:
|
||||
Initialized AlphaVantageProvider or None if API key not configured
|
||||
|
||||
Note: API key is fetched from system_settings database
|
||||
"""
|
||||
global _alphavantage_provider
|
||||
if _alphavantage_provider is not None:
|
||||
return _alphavantage_provider
|
||||
|
||||
settings_client = get_settings_client()
|
||||
api_key = await settings_client.get_api_key("alphavantage")
|
||||
|
||||
if not api_key:
|
||||
logger.warning("Alpha Vantage API key not configured - financial provider disabled")
|
||||
return None
|
||||
|
||||
_alphavantage_provider = AlphaVantageProvider(api_key=api_key)
|
||||
logger.debug("Created Alpha Vantage financial provider")
|
||||
return _alphavantage_provider
|
||||
|
||||
|
||||
# Type aliases for FastAPI endpoint dependencies
|
||||
# Usage: def my_endpoint(neo4j: Neo4jDep):
|
||||
Neo4jDep = Annotated[Neo4jClient, Depends(get_neo4j_client)]
|
||||
@@ -188,6 +319,12 @@ OllamaDep = Annotated[OllamaClient, Depends(get_ollama_client)]
|
||||
RedisDep = Annotated[aioredis.Redis, Depends(get_redis_client)]
|
||||
ContentExtractorDep = Annotated[ContentExtractor, Depends(get_content_extractor)]
|
||||
PaperlessDep = Annotated[PaperlessClient, Depends(get_paperless_client)]
|
||||
SettingsClientDep = Annotated[SettingsClient, Depends(get_settings_client)]
|
||||
|
||||
# External API provider dependencies
|
||||
WeatherProviderDep = Annotated[OpenMeteoProvider, Depends(get_weather_provider)]
|
||||
NewsProviderDep = Annotated[AggregatedNewsProvider, Depends(get_news_provider)]
|
||||
AlphaVantageProviderDep = Annotated[AlphaVantageProvider | None, Depends(get_alphavantage_provider)]
|
||||
|
||||
|
||||
# Lifecycle management functions
|
||||
@@ -241,6 +378,20 @@ async def startup_clients():
|
||||
else:
|
||||
logger.info("○ Paperless-ngx not configured (document storage disabled)")
|
||||
|
||||
# Check System Settings database availability
|
||||
if settings.system_settings_password:
|
||||
try:
|
||||
settings_client = get_settings_client()
|
||||
is_healthy = await settings_client.health_check()
|
||||
if is_healthy:
|
||||
logger.info(f"✓ System settings DB ready: {settings.system_settings_host}")
|
||||
else:
|
||||
logger.warning("✗ System settings DB not responding")
|
||||
except Exception as e:
|
||||
logger.error(f"✗ System settings health check failed: {e}")
|
||||
else:
|
||||
logger.info("○ System settings not configured")
|
||||
|
||||
# Qdrant, Wiki.js, SearXNG are lazy-initialized
|
||||
logger.info("Service clients startup complete")
|
||||
|
||||
@@ -271,6 +422,7 @@ async def shutdown_clients():
|
||||
("SearXNG", get_searxng_client()),
|
||||
("Ollama", get_ollama_client()),
|
||||
("Paperless", get_paperless_client()),
|
||||
("OpenMeteo", get_weather_provider()),
|
||||
]
|
||||
|
||||
for name, client in clients_to_close:
|
||||
@@ -280,6 +432,35 @@ async def shutdown_clients():
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing {name} client: {e}")
|
||||
|
||||
# Close async-initialized providers
|
||||
global _news_provider, _alphavantage_provider
|
||||
|
||||
if _news_provider is not None:
|
||||
try:
|
||||
await _news_provider.close()
|
||||
_news_provider = None
|
||||
logger.info("✓ News provider closed")
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing News provider: {e}")
|
||||
|
||||
if _alphavantage_provider is not None:
|
||||
try:
|
||||
await _alphavantage_provider.close()
|
||||
_alphavantage_provider = None
|
||||
logger.info("✓ AlphaVantage client closed")
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing AlphaVantage client: {e}")
|
||||
|
||||
# Close settings database connection
|
||||
settings = get_settings()
|
||||
if settings.system_settings_password:
|
||||
try:
|
||||
settings_client = get_settings_client()
|
||||
await settings_client.close()
|
||||
logger.info("✓ System settings client closed")
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing settings client: {e}")
|
||||
|
||||
logger.info("Service clients shutdown complete")
|
||||
|
||||
|
||||
@@ -364,6 +545,17 @@ async def check_service_health() -> dict:
|
||||
else:
|
||||
health["paperless"] = None # Not configured
|
||||
|
||||
# System Settings database
|
||||
if settings.system_settings_password:
|
||||
try:
|
||||
settings_client = get_settings_client()
|
||||
health["system_settings"] = await settings_client.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"System settings health check failed: {e}")
|
||||
health["system_settings"] = False
|
||||
else:
|
||||
health["system_settings"] = None # Not configured
|
||||
|
||||
return health
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user