feat: add external API providers and central settings database

- Add central settings database client (system_settings on postgres-shared)
  - User-scoped settings with global fallback
  - API config storage with enabled/disabled toggle
  - Per-source category filtering for news

- Add modular external API providers in src/apis/:
  - OpenMeteoProvider: weather with geocoding (free, no key)
  - NOSProvider: Dutch news RSS feeds
  - BBCProvider: English news RSS feeds
  - AggregatedNewsProvider: merges sources with category filtering
  - AlphaVantageProvider: financial quotes (API key from settings DB)

- Add provider dependencies and lifecycle management
- Add requirements-dev.txt with pip-audit for security auditing
- Add MEMORY_REMEMBER_PLAN.md documenting volatile/document memory architecture

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2025-12-26 12:07:58 +01:00
co-authored by Claude Opus 4.5
parent 99eefa291c
commit 5d4a8dba95
14 changed files with 2513 additions and 4 deletions
+192
View File
@@ -23,6 +23,12 @@ from src.clients.searxng_client import SearXNGClient
from src.clients.ollama_client import OllamaClient
from src.clients.content_extractor import ContentExtractor
from src.clients.paperless_client import PaperlessClient
from src.clients.settings_client import SettingsClient
from src.apis import (
OpenMeteoProvider,
AggregatedNewsProvider,
AlphaVantageProvider,
)
logger = logging.getLogger(__name__)
@@ -178,6 +184,131 @@ def get_paperless_client() -> PaperlessClient:
return client
@lru_cache
def get_settings_client() -> SettingsClient:
"""
Get central settings database client singleton.
Returns:
Initialized SettingsClient for Tatlock system_settings database
Note: Returns client with empty DSN if password not configured
"""
settings = get_settings()
if not settings.system_settings_password:
logger.warning("System settings password not configured - settings database disabled")
client = SettingsClient(dsn=settings.system_settings_dsn)
logger.debug(f"Created Settings client: {settings.system_settings_host}")
return client
# =============================================================================
# External API Providers
# =============================================================================
@lru_cache
def get_weather_provider() -> OpenMeteoProvider:
"""
Get Open-Meteo weather provider singleton.
Returns:
Initialized OpenMeteoProvider with default timezone
Note: Timezone can be overridden per-request for user preferences
"""
provider = OpenMeteoProvider(timezone="Europe/Amsterdam")
logger.debug("Created OpenMeteo weather provider")
return provider
# News provider requires sources from settings database
_news_provider: AggregatedNewsProvider | None = None
async def get_news_provider() -> AggregatedNewsProvider:
"""
Get aggregated news provider.
Returns:
Initialized AggregatedNewsProvider with user-configured sources
and per-source category filters.
Note: Configuration is fetched from system_settings database:
- news.sources: list of enabled sources (default: ["nos", "bbc"])
- api.{source}.categories: list of enabled categories per source
"""
global _news_provider
if _news_provider is not None:
return _news_provider
settings_client = get_settings_client()
# Get enabled sources
sources = await settings_client.get("news.sources")
if not sources or not isinstance(sources, list):
sources = ["nos", "bbc"]
logger.info(f"Using default news sources: {sources}")
else:
logger.info(f"Using configured news sources: {sources}")
# Filter out disabled sources and get category filters
enabled_sources: list[str] = []
category_filters: dict[str, list[str]] = {}
for source in sources:
config = await settings_client.get_api_config(source)
if config:
# Check if source is disabled
if config.get("enabled") is False:
logger.info(f"News source '{source}' is disabled - skipping")
continue
# Get category filter if specified
categories = config.get("categories", [])
if categories:
category_filters[source] = categories
logger.debug(f"Source '{source}' categories: {categories}")
enabled_sources.append(source)
if not enabled_sources:
enabled_sources = ["nos", "bbc"]
logger.warning("No enabled news sources - using defaults")
_news_provider = AggregatedNewsProvider(
sources=enabled_sources,
category_filters=category_filters
)
return _news_provider
# AlphaVantage requires API key from settings database
_alphavantage_provider: AlphaVantageProvider | None = None
async def get_alphavantage_provider() -> AlphaVantageProvider | None:
"""
Get Alpha Vantage financial provider.
Returns:
Initialized AlphaVantageProvider or None if API key not configured
Note: API key is fetched from system_settings database
"""
global _alphavantage_provider
if _alphavantage_provider is not None:
return _alphavantage_provider
settings_client = get_settings_client()
api_key = await settings_client.get_api_key("alphavantage")
if not api_key:
logger.warning("Alpha Vantage API key not configured - financial provider disabled")
return None
_alphavantage_provider = AlphaVantageProvider(api_key=api_key)
logger.debug("Created Alpha Vantage financial provider")
return _alphavantage_provider
# Type aliases for FastAPI endpoint dependencies
# Usage: def my_endpoint(neo4j: Neo4jDep):
Neo4jDep = Annotated[Neo4jClient, Depends(get_neo4j_client)]
@@ -188,6 +319,12 @@ OllamaDep = Annotated[OllamaClient, Depends(get_ollama_client)]
RedisDep = Annotated[aioredis.Redis, Depends(get_redis_client)]
ContentExtractorDep = Annotated[ContentExtractor, Depends(get_content_extractor)]
PaperlessDep = Annotated[PaperlessClient, Depends(get_paperless_client)]
SettingsClientDep = Annotated[SettingsClient, Depends(get_settings_client)]
# External API provider dependencies
WeatherProviderDep = Annotated[OpenMeteoProvider, Depends(get_weather_provider)]
NewsProviderDep = Annotated[AggregatedNewsProvider, Depends(get_news_provider)]
AlphaVantageProviderDep = Annotated[AlphaVantageProvider | None, Depends(get_alphavantage_provider)]
# Lifecycle management functions
@@ -241,6 +378,20 @@ async def startup_clients():
else:
logger.info("○ Paperless-ngx not configured (document storage disabled)")
# Check System Settings database availability
if settings.system_settings_password:
try:
settings_client = get_settings_client()
is_healthy = await settings_client.health_check()
if is_healthy:
logger.info(f"✓ System settings DB ready: {settings.system_settings_host}")
else:
logger.warning("✗ System settings DB not responding")
except Exception as e:
logger.error(f"✗ System settings health check failed: {e}")
else:
logger.info("○ System settings not configured")
# Qdrant, Wiki.js, SearXNG are lazy-initialized
logger.info("Service clients startup complete")
@@ -271,6 +422,7 @@ async def shutdown_clients():
("SearXNG", get_searxng_client()),
("Ollama", get_ollama_client()),
("Paperless", get_paperless_client()),
("OpenMeteo", get_weather_provider()),
]
for name, client in clients_to_close:
@@ -280,6 +432,35 @@ async def shutdown_clients():
except Exception as e:
logger.error(f"Error closing {name} client: {e}")
# Close async-initialized providers
global _news_provider, _alphavantage_provider
if _news_provider is not None:
try:
await _news_provider.close()
_news_provider = None
logger.info("✓ News provider closed")
except Exception as e:
logger.error(f"Error closing News provider: {e}")
if _alphavantage_provider is not None:
try:
await _alphavantage_provider.close()
_alphavantage_provider = None
logger.info("✓ AlphaVantage client closed")
except Exception as e:
logger.error(f"Error closing AlphaVantage client: {e}")
# Close settings database connection
settings = get_settings()
if settings.system_settings_password:
try:
settings_client = get_settings_client()
await settings_client.close()
logger.info("✓ System settings client closed")
except Exception as e:
logger.error(f"Error closing settings client: {e}")
logger.info("Service clients shutdown complete")
@@ -364,6 +545,17 @@ async def check_service_health() -> dict:
else:
health["paperless"] = None # Not configured
# System Settings database
if settings.system_settings_password:
try:
settings_client = get_settings_client()
health["system_settings"] = await settings_client.health_check()
except Exception as e:
logger.error(f"System settings health check failed: {e}")
health["system_settings"] = False
else:
health["system_settings"] = None # Not configured
return health