feat: add external API providers and central settings database
- Add central settings database client (system_settings on postgres-shared) - User-scoped settings with global fallback - API config storage with enabled/disabled toggle - Per-source category filtering for news - Add modular external API providers in src/apis/: - OpenMeteoProvider: weather with geocoding (free, no key) - NOSProvider: Dutch news RSS feeds - BBCProvider: English news RSS feeds - AggregatedNewsProvider: merges sources with category filtering - AlphaVantageProvider: financial quotes (API key from settings DB) - Add provider dependencies and lifecycle management - Add requirements-dev.txt with pip-audit for security auditing - Add MEMORY_REMEMBER_PLAN.md documenting volatile/document memory architecture 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
External API clients for Library Desk.
|
||||
|
||||
This package contains clients for external web APIs, named by source.
|
||||
Each provider implements a common interface for interoperability.
|
||||
|
||||
Weather providers (implement WeatherProvider):
|
||||
- openmeteo: Open-Meteo (free, no key)
|
||||
|
||||
News providers (implement NewsProvider):
|
||||
- nos: NOS.nl Dutch RSS (free, no key)
|
||||
- bbc: BBC English RSS (free, no key)
|
||||
|
||||
Financial providers (implement FinancialProvider):
|
||||
- alphavantage: Alpha Vantage (free tier with key)
|
||||
|
||||
Users can swap providers by configuring which implementation to use.
|
||||
All providers return standardized response models from base.py.
|
||||
"""
|
||||
|
||||
# Base classes and models
|
||||
from .base import (
|
||||
# Enums
|
||||
WeatherCondition,
|
||||
# Weather models
|
||||
CurrentWeather,
|
||||
DayForecast,
|
||||
WeatherForecast,
|
||||
GeoLocation,
|
||||
# News models
|
||||
NewsItem,
|
||||
NewsFeed,
|
||||
# Financial models
|
||||
StockQuote,
|
||||
# Abstract providers
|
||||
WeatherProvider,
|
||||
NewsProvider,
|
||||
FinancialProvider,
|
||||
)
|
||||
|
||||
# Concrete implementations
|
||||
from .openmeteo import OpenMeteoProvider
|
||||
from .nos import NOSProvider
|
||||
from .bbc import BBCProvider
|
||||
from .news import AggregatedNewsProvider
|
||||
from .alphavantage import AlphaVantageProvider
|
||||
|
||||
__all__ = [
|
||||
# Enums
|
||||
"WeatherCondition",
|
||||
# Weather
|
||||
"CurrentWeather",
|
||||
"DayForecast",
|
||||
"WeatherForecast",
|
||||
"GeoLocation",
|
||||
"WeatherProvider",
|
||||
"OpenMeteoProvider",
|
||||
# News
|
||||
"NewsItem",
|
||||
"NewsFeed",
|
||||
"NewsProvider",
|
||||
"NOSProvider",
|
||||
"BBCProvider",
|
||||
"AggregatedNewsProvider",
|
||||
# Financial
|
||||
"StockQuote",
|
||||
"FinancialProvider",
|
||||
"AlphaVantageProvider",
|
||||
]
|
||||
@@ -0,0 +1,227 @@
|
||||
"""
|
||||
Alpha Vantage financial API client.
|
||||
|
||||
Stock and cryptocurrency quotes.
|
||||
https://www.alphavantage.co/documentation/
|
||||
|
||||
Requires API key (free tier available).
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from .base import FinancialProvider, StockQuote
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AlphaVantageProvider(FinancialProvider):
|
||||
"""Alpha Vantage financial API implementation."""
|
||||
|
||||
BASE_URL = "https://www.alphavantage.co/query"
|
||||
|
||||
def __init__(self, api_key: str, timeout: int = 10):
|
||||
"""
|
||||
Initialize Alpha Vantage client.
|
||||
|
||||
Args:
|
||||
api_key: Alpha Vantage API key
|
||||
timeout: HTTP request timeout in seconds
|
||||
"""
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
@property
|
||||
def client(self) -> httpx.AsyncClient:
|
||||
"""Lazy-initialize HTTP client."""
|
||||
if self._client is None or self._client.is_closed:
|
||||
self._client = httpx.AsyncClient(timeout=self.timeout)
|
||||
return self._client
|
||||
|
||||
async def close(self):
|
||||
"""Close HTTP client."""
|
||||
if self._client and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def get_quote(self, symbol: str) -> Optional[StockQuote]:
|
||||
"""
|
||||
Get current quote for a stock symbol.
|
||||
|
||||
Args:
|
||||
symbol: Stock ticker symbol (e.g., "AAPL", "MSFT")
|
||||
|
||||
Returns:
|
||||
StockQuote with current price info or None if not found
|
||||
"""
|
||||
try:
|
||||
response = await self.client.get(
|
||||
self.BASE_URL,
|
||||
params={
|
||||
"function": "GLOBAL_QUOTE",
|
||||
"symbol": symbol.upper(),
|
||||
"apikey": self.api_key
|
||||
}
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
# Check for API errors
|
||||
if "Error Message" in data:
|
||||
logger.warning(f"Alpha Vantage error for {symbol}: {data['Error Message']}")
|
||||
return None
|
||||
|
||||
if "Note" in data:
|
||||
# Rate limit warning
|
||||
logger.warning(f"Alpha Vantage rate limit: {data['Note']}")
|
||||
return None
|
||||
|
||||
quote = data.get("Global Quote", {})
|
||||
if not quote:
|
||||
logger.warning(f"No quote data for symbol: {symbol}")
|
||||
return None
|
||||
|
||||
# Parse quote data
|
||||
price = float(quote.get("05. price", 0))
|
||||
change = float(quote.get("09. change", 0))
|
||||
change_percent_str = quote.get("10. change percent", "0%")
|
||||
change_percent = float(change_percent_str.rstrip('%'))
|
||||
|
||||
return StockQuote(
|
||||
symbol=symbol.upper(),
|
||||
name=None, # Global Quote doesn't include company name
|
||||
price=price,
|
||||
currency="USD", # Alpha Vantage returns USD for US stocks
|
||||
change=change,
|
||||
change_percent=change_percent,
|
||||
timestamp=datetime.now()
|
||||
)
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Alpha Vantage request failed for {symbol}: {e}")
|
||||
return None
|
||||
except (KeyError, ValueError) as e:
|
||||
logger.error(f"Failed to parse Alpha Vantage response for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
async def get_quotes(self, symbols: list[str]) -> list[StockQuote]:
|
||||
"""
|
||||
Get quotes for multiple stock symbols.
|
||||
|
||||
Note: Alpha Vantage free tier has rate limits (5 calls/min, 500 calls/day).
|
||||
Consider using batch endpoints or caching for production use.
|
||||
|
||||
Args:
|
||||
symbols: List of stock ticker symbols
|
||||
|
||||
Returns:
|
||||
List of StockQuote objects (may be less than input if some fail)
|
||||
"""
|
||||
quotes = []
|
||||
for symbol in symbols:
|
||||
quote = await self.get_quote(symbol)
|
||||
if quote:
|
||||
quotes.append(quote)
|
||||
return quotes
|
||||
|
||||
async def get_crypto_quote(
|
||||
self,
|
||||
symbol: str,
|
||||
market: str = "USD"
|
||||
) -> Optional[StockQuote]:
|
||||
"""
|
||||
Get current quote for a cryptocurrency.
|
||||
|
||||
Args:
|
||||
symbol: Crypto symbol (e.g., "BTC", "ETH")
|
||||
market: Market currency (default: USD)
|
||||
|
||||
Returns:
|
||||
StockQuote with current price info or None if not found
|
||||
"""
|
||||
try:
|
||||
response = await self.client.get(
|
||||
self.BASE_URL,
|
||||
params={
|
||||
"function": "CURRENCY_EXCHANGE_RATE",
|
||||
"from_currency": symbol.upper(),
|
||||
"to_currency": market.upper(),
|
||||
"apikey": self.api_key
|
||||
}
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
# Check for API errors
|
||||
if "Error Message" in data:
|
||||
logger.warning(f"Alpha Vantage error for {symbol}: {data['Error Message']}")
|
||||
return None
|
||||
|
||||
if "Note" in data:
|
||||
logger.warning(f"Alpha Vantage rate limit: {data['Note']}")
|
||||
return None
|
||||
|
||||
rate_data = data.get("Realtime Currency Exchange Rate", {})
|
||||
if not rate_data:
|
||||
logger.warning(f"No exchange rate data for: {symbol}/{market}")
|
||||
return None
|
||||
|
||||
price = float(rate_data.get("5. Exchange Rate", 0))
|
||||
|
||||
return StockQuote(
|
||||
symbol=f"{symbol.upper()}/{market.upper()}",
|
||||
name=rate_data.get("2. From_Currency Name"),
|
||||
price=price,
|
||||
currency=market.upper(),
|
||||
change=None, # Exchange rate endpoint doesn't provide change
|
||||
change_percent=None,
|
||||
timestamp=datetime.now()
|
||||
)
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Alpha Vantage crypto request failed for {symbol}: {e}")
|
||||
return None
|
||||
except (KeyError, ValueError) as e:
|
||||
logger.error(f"Failed to parse Alpha Vantage crypto response for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
async def search_symbol(self, keywords: str) -> list[dict]:
|
||||
"""
|
||||
Search for stock symbols by keywords.
|
||||
|
||||
Args:
|
||||
keywords: Search keywords (company name or partial symbol)
|
||||
|
||||
Returns:
|
||||
List of matching symbols with metadata
|
||||
"""
|
||||
try:
|
||||
response = await self.client.get(
|
||||
self.BASE_URL,
|
||||
params={
|
||||
"function": "SYMBOL_SEARCH",
|
||||
"keywords": keywords,
|
||||
"apikey": self.api_key
|
||||
}
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
matches = data.get("bestMatches", [])
|
||||
return [
|
||||
{
|
||||
"symbol": m.get("1. symbol"),
|
||||
"name": m.get("2. name"),
|
||||
"type": m.get("3. type"),
|
||||
"region": m.get("4. region"),
|
||||
"currency": m.get("8. currency"),
|
||||
}
|
||||
for m in matches
|
||||
]
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Alpha Vantage search failed for '{keywords}': {e}")
|
||||
return []
|
||||
@@ -0,0 +1,215 @@
|
||||
"""
|
||||
Base classes and standardized response models for external APIs.
|
||||
|
||||
All provider implementations should return these standard models
|
||||
to ensure interoperability when swapping providers.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from enum import Enum
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Weather Models
|
||||
# =============================================================================
|
||||
|
||||
class WeatherCondition(Enum):
|
||||
"""Standardized weather conditions across providers."""
|
||||
CLEAR = "clear"
|
||||
PARTLY_CLOUDY = "partly_cloudy"
|
||||
CLOUDY = "cloudy"
|
||||
OVERCAST = "overcast"
|
||||
FOG = "fog"
|
||||
DRIZZLE = "drizzle"
|
||||
RAIN = "rain"
|
||||
HEAVY_RAIN = "heavy_rain"
|
||||
SNOW = "snow"
|
||||
HEAVY_SNOW = "heavy_snow"
|
||||
THUNDERSTORM = "thunderstorm"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CurrentWeather:
|
||||
"""Standardized current weather response."""
|
||||
temperature: float # Celsius
|
||||
feels_like: Optional[float] # Celsius
|
||||
humidity: int # Percentage 0-100
|
||||
wind_speed: float # km/h
|
||||
wind_direction: Optional[int] # Degrees 0-360
|
||||
condition: WeatherCondition
|
||||
condition_text: str # Human-readable description
|
||||
timestamp: datetime
|
||||
location: str # City/location name
|
||||
|
||||
def to_text(self) -> str:
|
||||
"""Generate natural language description."""
|
||||
return (
|
||||
f"Currently {self.temperature:.1f}°C "
|
||||
f"({self.condition_text}) in {self.location}. "
|
||||
f"Humidity {self.humidity}%, wind {self.wind_speed:.0f} km/h."
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DayForecast:
|
||||
"""Standardized daily forecast."""
|
||||
date: datetime
|
||||
temp_high: float # Celsius
|
||||
temp_low: float # Celsius
|
||||
condition: WeatherCondition
|
||||
condition_text: str
|
||||
precipitation_chance: Optional[int] # Percentage 0-100
|
||||
precipitation_mm: Optional[float]
|
||||
|
||||
|
||||
@dataclass
|
||||
class WeatherForecast:
|
||||
"""Standardized forecast response."""
|
||||
location: str
|
||||
current: CurrentWeather
|
||||
daily: list[DayForecast] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GeoLocation:
|
||||
"""Geocoding result."""
|
||||
name: str
|
||||
latitude: float
|
||||
longitude: float
|
||||
country: Optional[str] = None
|
||||
admin_area: Optional[str] = None # State/province
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# News Models
|
||||
# =============================================================================
|
||||
|
||||
@dataclass
|
||||
class NewsItem:
|
||||
"""Standardized news article/item."""
|
||||
title: str
|
||||
description: Optional[str]
|
||||
url: str
|
||||
published: Optional[datetime]
|
||||
source: str # e.g., "nos", "bbc"
|
||||
category: Optional[str] = None # e.g., "tech", "world"
|
||||
image_url: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class NewsFeed:
|
||||
"""Standardized news feed response."""
|
||||
source: str
|
||||
category: str
|
||||
items: list[NewsItem] = field(default_factory=list)
|
||||
fetched_at: datetime = field(default_factory=datetime.now)
|
||||
|
||||
def to_text(self) -> str:
|
||||
"""Generate natural language summary of headlines."""
|
||||
if not self.items:
|
||||
return f"No news available from {self.source}."
|
||||
|
||||
headlines = [f"- {item.title}" for item in self.items[:5]]
|
||||
return f"Headlines from {self.source} ({self.category}):\n" + "\n".join(headlines)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Financial Models
|
||||
# =============================================================================
|
||||
|
||||
@dataclass
|
||||
class StockQuote:
|
||||
"""Standardized stock/crypto quote."""
|
||||
symbol: str
|
||||
name: Optional[str]
|
||||
price: float
|
||||
currency: str # e.g., "USD", "EUR"
|
||||
change: Optional[float] # Absolute change
|
||||
change_percent: Optional[float] # Percentage change
|
||||
timestamp: datetime
|
||||
|
||||
def to_text(self) -> str:
|
||||
"""Generate natural language description."""
|
||||
change_str = ""
|
||||
if self.change is not None and self.change_percent is not None:
|
||||
direction = "up" if self.change >= 0 else "down"
|
||||
change_str = f", {direction} {abs(self.change_percent):.2f}%"
|
||||
return f"{self.symbol}: {self.price:.2f} {self.currency}{change_str}"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Provider Interfaces
|
||||
# =============================================================================
|
||||
|
||||
class WeatherProvider(ABC):
|
||||
"""Abstract base class for weather API providers."""
|
||||
|
||||
@abstractmethod
|
||||
async def geocode(self, city: str) -> Optional[GeoLocation]:
|
||||
"""Convert city name to coordinates."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_current(self, location: GeoLocation) -> CurrentWeather:
|
||||
"""Get current weather for a location."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_forecast(self, location: GeoLocation, days: int = 7) -> WeatherForecast:
|
||||
"""Get weather forecast for a location."""
|
||||
pass
|
||||
|
||||
async def get_weather_for_city(self, city: str) -> CurrentWeather:
|
||||
"""Convenience method: geocode and get current weather."""
|
||||
location = await self.geocode(city)
|
||||
if not location:
|
||||
raise ValueError(f"Could not geocode city: {city}")
|
||||
return await self.get_current(location)
|
||||
|
||||
|
||||
class NewsProvider(ABC):
|
||||
"""Abstract base class for news API providers."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def source_name(self) -> str:
|
||||
"""Provider name (e.g., 'nos', 'bbc')."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def available_categories(self) -> list[str]:
|
||||
"""List of available category keys."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_feed(self, category: str, limit: int = 10) -> NewsFeed:
|
||||
"""Get news feed for a category."""
|
||||
pass
|
||||
|
||||
async def get_headlines(self, categories: list[str], limit: int = 5) -> list[NewsFeed]:
|
||||
"""Get headlines from multiple categories."""
|
||||
feeds = []
|
||||
for cat in categories:
|
||||
if cat in self.available_categories:
|
||||
feed = await self.get_feed(cat, limit)
|
||||
feeds.append(feed)
|
||||
return feeds
|
||||
|
||||
|
||||
class FinancialProvider(ABC):
|
||||
"""Abstract base class for financial API providers."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_quote(self, symbol: str) -> Optional[StockQuote]:
|
||||
"""Get current quote for a stock/crypto symbol."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_quotes(self, symbols: list[str]) -> list[StockQuote]:
|
||||
"""Get quotes for multiple symbols."""
|
||||
pass
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
BBC News RSS client.
|
||||
|
||||
Free RSS feeds from BBC News.
|
||||
https://www.bbc.com/news/10628494 (RSS feed directory)
|
||||
|
||||
No API key required.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import feedparser
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Optional
|
||||
|
||||
from .base import NewsProvider, NewsItem, NewsFeed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BBCProvider(NewsProvider):
|
||||
"""BBC News RSS feed implementation."""
|
||||
|
||||
# Available BBC RSS feeds
|
||||
FEEDS: dict[str, str] = {
|
||||
# News
|
||||
"top": "https://feeds.bbci.co.uk/news/rss.xml",
|
||||
"world": "https://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
"uk": "https://feeds.bbci.co.uk/news/uk/rss.xml",
|
||||
"business": "https://feeds.bbci.co.uk/news/business/rss.xml",
|
||||
"politics": "https://feeds.bbci.co.uk/news/politics/rss.xml",
|
||||
"health": "https://feeds.bbci.co.uk/news/health/rss.xml",
|
||||
"education": "https://feeds.bbci.co.uk/news/education/rss.xml",
|
||||
"science": "https://feeds.bbci.co.uk/news/science_and_environment/rss.xml",
|
||||
"tech": "https://feeds.bbci.co.uk/news/technology/rss.xml",
|
||||
"entertainment": "https://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml",
|
||||
"asia": "https://feeds.bbci.co.uk/news/world/asia/rss.xml",
|
||||
"europe": "https://feeds.bbci.co.uk/news/world/europe/rss.xml",
|
||||
"africa": "https://feeds.bbci.co.uk/news/world/africa/rss.xml",
|
||||
# Sports
|
||||
"sports": "https://feeds.bbci.co.uk/sport/rss.xml",
|
||||
"football": "https://feeds.bbci.co.uk/sport/football/rss.xml",
|
||||
"cricket": "https://feeds.bbci.co.uk/sport/cricket/rss.xml",
|
||||
"tennis": "https://feeds.bbci.co.uk/sport/tennis/rss.xml",
|
||||
"rugby": "https://feeds.bbci.co.uk/sport/rugby-union/rss.xml",
|
||||
"f1": "https://feeds.bbci.co.uk/sport/motorsport/rss.xml",
|
||||
"golf": "https://feeds.bbci.co.uk/sport/golf/rss.xml",
|
||||
}
|
||||
|
||||
def __init__(self, timeout: int = 10):
|
||||
"""
|
||||
Initialize BBC RSS client.
|
||||
|
||||
Args:
|
||||
timeout: HTTP request timeout in seconds
|
||||
"""
|
||||
self.timeout = timeout
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
@property
|
||||
def client(self) -> httpx.AsyncClient:
|
||||
"""Lazy-initialize HTTP client."""
|
||||
if self._client is None or self._client.is_closed:
|
||||
self._client = httpx.AsyncClient(timeout=self.timeout)
|
||||
return self._client
|
||||
|
||||
async def close(self):
|
||||
"""Close HTTP client."""
|
||||
if self._client and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def source_name(self) -> str:
|
||||
"""Provider name."""
|
||||
return "bbc"
|
||||
|
||||
@property
|
||||
def available_categories(self) -> list[str]:
|
||||
"""List of available category keys."""
|
||||
return list(self.FEEDS.keys())
|
||||
|
||||
async def get_feed(self, category: str, limit: int = 10) -> NewsFeed:
|
||||
"""
|
||||
Get news feed for a category.
|
||||
|
||||
Args:
|
||||
category: Feed category (top, world, uk, business, etc.)
|
||||
limit: Maximum number of items to return
|
||||
|
||||
Returns:
|
||||
NewsFeed with standardized news items
|
||||
|
||||
Raises:
|
||||
ValueError: If category is not available
|
||||
"""
|
||||
if category not in self.FEEDS:
|
||||
raise ValueError(
|
||||
f"Unknown category '{category}'. "
|
||||
f"Available: {', '.join(self.available_categories)}"
|
||||
)
|
||||
|
||||
feed_url = self.FEEDS[category]
|
||||
|
||||
try:
|
||||
response = await self.client.get(feed_url)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse RSS feed
|
||||
feed = feedparser.parse(response.text)
|
||||
|
||||
items = []
|
||||
for entry in feed.entries[:limit]:
|
||||
# Parse publication date
|
||||
published = None
|
||||
if hasattr(entry, 'published'):
|
||||
try:
|
||||
published = parsedate_to_datetime(entry.published)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# BBC uses media:thumbnail for images
|
||||
image_url = None
|
||||
if hasattr(entry, 'media_thumbnail') and entry.media_thumbnail:
|
||||
image_url = entry.media_thumbnail[0].get('url')
|
||||
elif hasattr(entry, 'media_content') and entry.media_content:
|
||||
image_url = entry.media_content[0].get('url')
|
||||
|
||||
items.append(NewsItem(
|
||||
title=entry.get('title', 'No title'),
|
||||
description=entry.get('summary') or entry.get('description'),
|
||||
url=entry.get('link', ''),
|
||||
published=published,
|
||||
source=self.source_name,
|
||||
category=category,
|
||||
image_url=image_url
|
||||
))
|
||||
|
||||
return NewsFeed(
|
||||
source=self.source_name,
|
||||
category=category,
|
||||
items=items,
|
||||
fetched_at=datetime.now()
|
||||
)
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"BBC feed request failed for '{category}': {e}")
|
||||
raise ValueError(f"Failed to fetch BBC feed: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse BBC feed '{category}': {e}")
|
||||
raise ValueError(f"Failed to parse BBC feed: {e}")
|
||||
@@ -0,0 +1,241 @@
|
||||
"""
|
||||
Aggregated news provider.
|
||||
|
||||
Combines multiple news sources into a single chronologically-sorted stream.
|
||||
Source selection is driven by user preferences in the settings database.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from .base import NewsProvider, NewsItem, NewsFeed
|
||||
from .nos import NOSProvider
|
||||
from .bbc import BBCProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Registry of available news providers
|
||||
PROVIDER_REGISTRY: dict[str, type[NewsProvider]] = {
|
||||
"nos": NOSProvider,
|
||||
"bbc": BBCProvider,
|
||||
}
|
||||
|
||||
|
||||
class AggregatedNewsProvider:
|
||||
"""
|
||||
Aggregated news provider that combines multiple sources.
|
||||
|
||||
Fetches from configured sources in parallel and merges results
|
||||
into a single chronologically-sorted stream. Only fetches from
|
||||
enabled categories per source.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sources: list[str],
|
||||
category_filters: dict[str, list[str]] | None = None,
|
||||
timeout: int = 10
|
||||
):
|
||||
"""
|
||||
Initialize aggregated provider.
|
||||
|
||||
Args:
|
||||
sources: List of source names to aggregate (e.g., ["nos", "bbc"])
|
||||
category_filters: Per-source enabled categories.
|
||||
Example: {"nos": ["general", "tech"], "bbc": ["top", "world"]}
|
||||
Empty list or missing entry = all categories allowed.
|
||||
timeout: HTTP request timeout in seconds
|
||||
"""
|
||||
self.sources = sources
|
||||
self.category_filters = category_filters or {}
|
||||
self.timeout = timeout
|
||||
self._providers: dict[str, NewsProvider] = {}
|
||||
|
||||
# Initialize configured providers
|
||||
for source in sources:
|
||||
if source in PROVIDER_REGISTRY:
|
||||
self._providers[source] = PROVIDER_REGISTRY[source](timeout=timeout)
|
||||
else:
|
||||
logger.warning(f"Unknown news source '{source}' - skipping")
|
||||
|
||||
def _is_category_enabled(self, source: str, category: str) -> bool:
|
||||
"""Check if a category is enabled for a source."""
|
||||
allowed = self.category_filters.get(source, [])
|
||||
# Empty list = all allowed
|
||||
if not allowed:
|
||||
return True
|
||||
return category in allowed
|
||||
|
||||
def _get_enabled_categories(self, source: str) -> list[str]:
|
||||
"""Get list of enabled categories for a source."""
|
||||
provider = self._providers.get(source)
|
||||
if not provider:
|
||||
return []
|
||||
|
||||
allowed = self.category_filters.get(source, [])
|
||||
if not allowed:
|
||||
# All categories enabled
|
||||
return provider.available_categories
|
||||
|
||||
# Filter to only enabled ones that exist
|
||||
return [c for c in allowed if c in provider.available_categories]
|
||||
|
||||
@property
|
||||
def available_sources(self) -> list[str]:
|
||||
"""List of initialized source names."""
|
||||
return list(self._providers.keys())
|
||||
|
||||
@property
|
||||
def available_categories(self) -> dict[str, list[str]]:
|
||||
"""Map of source -> available categories."""
|
||||
return {
|
||||
name: provider.available_categories
|
||||
for name, provider in self._providers.items()
|
||||
}
|
||||
|
||||
def _normalize_timestamp(self, item: NewsItem) -> datetime:
|
||||
"""Get UTC timestamp for sorting, with fallback for missing timestamps."""
|
||||
if item.published:
|
||||
# Ensure UTC
|
||||
if item.published.tzinfo is None:
|
||||
return item.published.replace(tzinfo=timezone.utc)
|
||||
return item.published.astimezone(timezone.utc)
|
||||
# Fallback: use current time (item will sort to top)
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
async def get_feed(
|
||||
self,
|
||||
category: str = "general",
|
||||
limit: int = 20
|
||||
) -> NewsFeed:
|
||||
"""
|
||||
Get aggregated news feed from all sources.
|
||||
|
||||
Args:
|
||||
category: Category to fetch. Maps to source-specific categories:
|
||||
- "general"/"top": general news from all sources
|
||||
- "world": international news
|
||||
- "tech": technology news
|
||||
- "business"/"economy": business/economy news
|
||||
- "politics": political news
|
||||
limit: Maximum total items to return (after merging)
|
||||
|
||||
Returns:
|
||||
NewsFeed with merged, chronologically-sorted items
|
||||
"""
|
||||
# Map generic categories to source-specific ones
|
||||
category_map = {
|
||||
"nos": {
|
||||
"general": "general",
|
||||
"top": "general",
|
||||
"world": "world",
|
||||
"tech": "tech",
|
||||
"business": "economy",
|
||||
"economy": "economy",
|
||||
"politics": "politics",
|
||||
},
|
||||
"bbc": {
|
||||
"general": "top",
|
||||
"top": "top",
|
||||
"world": "world",
|
||||
"tech": "tech",
|
||||
"business": "business",
|
||||
"economy": "business",
|
||||
"politics": "politics",
|
||||
},
|
||||
}
|
||||
|
||||
# Fetch from all sources in parallel
|
||||
async def fetch_source(name: str, provider: NewsProvider) -> list[NewsItem]:
|
||||
try:
|
||||
source_category = category_map.get(name, {}).get(category, category)
|
||||
if source_category not in provider.available_categories:
|
||||
logger.debug(f"Category '{category}' not available for {name}")
|
||||
return []
|
||||
# Check if category is enabled for this source
|
||||
if not self._is_category_enabled(name, source_category):
|
||||
logger.debug(f"Category '{source_category}' disabled for {name}")
|
||||
return []
|
||||
feed = await provider.get_feed(source_category, limit=limit)
|
||||
return feed.items
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch from {name}: {e}")
|
||||
return []
|
||||
|
||||
tasks = [
|
||||
fetch_source(name, provider)
|
||||
for name, provider in self._providers.items()
|
||||
]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
# Merge all items
|
||||
all_items: list[NewsItem] = []
|
||||
for items in results:
|
||||
all_items.extend(items)
|
||||
|
||||
# Sort by timestamp (newest first)
|
||||
all_items.sort(key=self._normalize_timestamp, reverse=True)
|
||||
|
||||
# Apply limit
|
||||
all_items = all_items[:limit]
|
||||
|
||||
return NewsFeed(
|
||||
source="aggregated",
|
||||
category=category,
|
||||
items=all_items,
|
||||
fetched_at=datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
async def get_headlines(
|
||||
self,
|
||||
categories: list[str] | None = None,
|
||||
limit: int = 10
|
||||
) -> NewsFeed:
|
||||
"""
|
||||
Get headlines from multiple categories, merged into one feed.
|
||||
|
||||
Args:
|
||||
categories: Categories to fetch. If None, fetches from all
|
||||
enabled categories across all sources.
|
||||
limit: Maximum total items to return
|
||||
|
||||
Returns:
|
||||
NewsFeed with merged headlines from all categories
|
||||
"""
|
||||
if categories is None:
|
||||
# Collect all enabled categories across sources
|
||||
all_categories: set[str] = set()
|
||||
for source in self._providers:
|
||||
all_categories.update(self._get_enabled_categories(source))
|
||||
categories = list(all_categories) if all_categories else ["general"]
|
||||
|
||||
# Fetch all categories
|
||||
tasks = [self.get_feed(cat, limit=limit) for cat in categories]
|
||||
feeds = await asyncio.gather(*tasks)
|
||||
|
||||
# Merge and deduplicate by URL
|
||||
seen_urls: set[str] = set()
|
||||
all_items: list[NewsItem] = []
|
||||
|
||||
for feed in feeds:
|
||||
for item in feed.items:
|
||||
if item.url not in seen_urls:
|
||||
seen_urls.add(item.url)
|
||||
all_items.append(item)
|
||||
|
||||
# Sort by timestamp
|
||||
all_items.sort(key=self._normalize_timestamp, reverse=True)
|
||||
|
||||
return NewsFeed(
|
||||
source="aggregated",
|
||||
category=",".join(categories),
|
||||
items=all_items[:limit],
|
||||
fetched_at=datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
async def close(self):
|
||||
"""Close all provider HTTP clients."""
|
||||
for provider in self._providers.values():
|
||||
await provider.close()
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
"""
|
||||
NOS.nl Dutch news RSS client.
|
||||
|
||||
Free RSS feeds from Netherlands public broadcaster.
|
||||
https://nos.nl/feeds
|
||||
|
||||
No API key required.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import feedparser
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Optional
|
||||
|
||||
from .base import NewsProvider, NewsItem, NewsFeed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NOSProvider(NewsProvider):
|
||||
"""NOS.nl RSS feed implementation."""
|
||||
|
||||
# Available NOS RSS feeds
|
||||
FEEDS: dict[str, str] = {
|
||||
# News
|
||||
"general": "https://feeds.nos.nl/nosnieuwsalgemeen",
|
||||
"domestic": "https://feeds.nos.nl/nosnieuwsbinnenland",
|
||||
"world": "https://feeds.nos.nl/nosnieuwsbuitenland",
|
||||
"politics": "https://feeds.nos.nl/nosnieuwspolitiek",
|
||||
"economy": "https://feeds.nos.nl/nosnieuwseconomie",
|
||||
"remarkable": "https://feeds.nos.nl/nosnieuwsopmerkelijk",
|
||||
"culture": "https://feeds.nos.nl/nosnieuwscultuurenmedia",
|
||||
"tech": "https://feeds.nos.nl/nosnieuwstech",
|
||||
# Sports
|
||||
"sports": "https://feeds.nos.nl/nossportalgemeen",
|
||||
"football": "https://feeds.nos.nl/nosvoetbal",
|
||||
"cycling": "https://feeds.nos.nl/nossportwielrennen",
|
||||
"skating": "https://feeds.nos.nl/nossportschaatsen",
|
||||
"tennis": "https://feeds.nos.nl/nossporttennis",
|
||||
"f1": "https://feeds.nos.nl/nossportformule1",
|
||||
}
|
||||
|
||||
def __init__(self, timeout: int = 10):
|
||||
"""
|
||||
Initialize NOS RSS client.
|
||||
|
||||
Args:
|
||||
timeout: HTTP request timeout in seconds
|
||||
"""
|
||||
self.timeout = timeout
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
@property
|
||||
def client(self) -> httpx.AsyncClient:
|
||||
"""Lazy-initialize HTTP client."""
|
||||
if self._client is None or self._client.is_closed:
|
||||
self._client = httpx.AsyncClient(timeout=self.timeout)
|
||||
return self._client
|
||||
|
||||
async def close(self):
|
||||
"""Close HTTP client."""
|
||||
if self._client and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def source_name(self) -> str:
|
||||
"""Provider name."""
|
||||
return "nos"
|
||||
|
||||
@property
|
||||
def available_categories(self) -> list[str]:
|
||||
"""List of available category keys."""
|
||||
return list(self.FEEDS.keys())
|
||||
|
||||
async def get_feed(self, category: str, limit: int = 10) -> NewsFeed:
|
||||
"""
|
||||
Get news feed for a category.
|
||||
|
||||
Args:
|
||||
category: Feed category (general, domestic, world, etc.)
|
||||
limit: Maximum number of items to return
|
||||
|
||||
Returns:
|
||||
NewsFeed with standardized news items
|
||||
|
||||
Raises:
|
||||
ValueError: If category is not available
|
||||
"""
|
||||
if category not in self.FEEDS:
|
||||
raise ValueError(
|
||||
f"Unknown category '{category}'. "
|
||||
f"Available: {', '.join(self.available_categories)}"
|
||||
)
|
||||
|
||||
feed_url = self.FEEDS[category]
|
||||
|
||||
try:
|
||||
response = await self.client.get(feed_url)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse RSS feed
|
||||
feed = feedparser.parse(response.text)
|
||||
|
||||
items = []
|
||||
for entry in feed.entries[:limit]:
|
||||
# Parse publication date
|
||||
published = None
|
||||
if hasattr(entry, 'published'):
|
||||
try:
|
||||
published = parsedate_to_datetime(entry.published)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# Extract image URL if available
|
||||
image_url = None
|
||||
if hasattr(entry, 'media_content') and entry.media_content:
|
||||
image_url = entry.media_content[0].get('url')
|
||||
elif hasattr(entry, 'enclosures') and entry.enclosures:
|
||||
for enc in entry.enclosures:
|
||||
if enc.get('type', '').startswith('image/'):
|
||||
image_url = enc.get('href')
|
||||
break
|
||||
|
||||
items.append(NewsItem(
|
||||
title=entry.get('title', 'No title'),
|
||||
description=entry.get('summary') or entry.get('description'),
|
||||
url=entry.get('link', ''),
|
||||
published=published,
|
||||
source=self.source_name,
|
||||
category=category,
|
||||
image_url=image_url
|
||||
))
|
||||
|
||||
return NewsFeed(
|
||||
source=self.source_name,
|
||||
category=category,
|
||||
items=items,
|
||||
fetched_at=datetime.now()
|
||||
)
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"NOS feed request failed for '{category}': {e}")
|
||||
raise ValueError(f"Failed to fetch NOS feed: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse NOS feed '{category}': {e}")
|
||||
raise ValueError(f"Failed to parse NOS feed: {e}")
|
||||
@@ -0,0 +1,311 @@
|
||||
"""
|
||||
Open-Meteo weather API client.
|
||||
|
||||
Free weather API with no API key required.
|
||||
https://open-meteo.com/en/docs
|
||||
|
||||
Uses Open-Meteo Geocoding API for city name to coordinate conversion.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from .base import (
|
||||
WeatherProvider,
|
||||
WeatherCondition,
|
||||
CurrentWeather,
|
||||
DayForecast,
|
||||
WeatherForecast,
|
||||
GeoLocation,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# WMO Weather interpretation codes to our standardized conditions
|
||||
# https://open-meteo.com/en/docs#weathervariables
|
||||
WMO_CODE_MAP: dict[int, WeatherCondition] = {
|
||||
0: WeatherCondition.CLEAR, # Clear sky
|
||||
1: WeatherCondition.CLEAR, # Mainly clear
|
||||
2: WeatherCondition.PARTLY_CLOUDY, # Partly cloudy
|
||||
3: WeatherCondition.CLOUDY, # Overcast
|
||||
45: WeatherCondition.FOG, # Fog
|
||||
48: WeatherCondition.FOG, # Depositing rime fog
|
||||
51: WeatherCondition.DRIZZLE, # Light drizzle
|
||||
53: WeatherCondition.DRIZZLE, # Moderate drizzle
|
||||
55: WeatherCondition.DRIZZLE, # Dense drizzle
|
||||
56: WeatherCondition.DRIZZLE, # Light freezing drizzle
|
||||
57: WeatherCondition.DRIZZLE, # Dense freezing drizzle
|
||||
61: WeatherCondition.RAIN, # Slight rain
|
||||
63: WeatherCondition.RAIN, # Moderate rain
|
||||
65: WeatherCondition.HEAVY_RAIN, # Heavy rain
|
||||
66: WeatherCondition.RAIN, # Light freezing rain
|
||||
67: WeatherCondition.HEAVY_RAIN, # Heavy freezing rain
|
||||
71: WeatherCondition.SNOW, # Slight snow fall
|
||||
73: WeatherCondition.SNOW, # Moderate snow fall
|
||||
75: WeatherCondition.HEAVY_SNOW, # Heavy snow fall
|
||||
77: WeatherCondition.SNOW, # Snow grains
|
||||
80: WeatherCondition.RAIN, # Slight rain showers
|
||||
81: WeatherCondition.RAIN, # Moderate rain showers
|
||||
82: WeatherCondition.HEAVY_RAIN, # Violent rain showers
|
||||
85: WeatherCondition.SNOW, # Slight snow showers
|
||||
86: WeatherCondition.HEAVY_SNOW, # Heavy snow showers
|
||||
95: WeatherCondition.THUNDERSTORM, # Thunderstorm
|
||||
96: WeatherCondition.THUNDERSTORM, # Thunderstorm with slight hail
|
||||
99: WeatherCondition.THUNDERSTORM, # Thunderstorm with heavy hail
|
||||
}
|
||||
|
||||
# Human-readable descriptions for WMO codes
|
||||
WMO_DESCRIPTIONS: dict[int, str] = {
|
||||
0: "Clear sky",
|
||||
1: "Mainly clear",
|
||||
2: "Partly cloudy",
|
||||
3: "Overcast",
|
||||
45: "Fog",
|
||||
48: "Depositing rime fog",
|
||||
51: "Light drizzle",
|
||||
53: "Moderate drizzle",
|
||||
55: "Dense drizzle",
|
||||
56: "Light freezing drizzle",
|
||||
57: "Dense freezing drizzle",
|
||||
61: "Slight rain",
|
||||
63: "Moderate rain",
|
||||
65: "Heavy rain",
|
||||
66: "Light freezing rain",
|
||||
67: "Heavy freezing rain",
|
||||
71: "Slight snow fall",
|
||||
73: "Moderate snow fall",
|
||||
75: "Heavy snow fall",
|
||||
77: "Snow grains",
|
||||
80: "Slight rain showers",
|
||||
81: "Moderate rain showers",
|
||||
82: "Violent rain showers",
|
||||
85: "Slight snow showers",
|
||||
86: "Heavy snow showers",
|
||||
95: "Thunderstorm",
|
||||
96: "Thunderstorm with slight hail",
|
||||
99: "Thunderstorm with heavy hail",
|
||||
}
|
||||
|
||||
|
||||
class OpenMeteoProvider(WeatherProvider):
|
||||
"""Open-Meteo weather API implementation."""
|
||||
|
||||
GEOCODING_URL = "https://geocoding-api.open-meteo.com/v1/search"
|
||||
WEATHER_URL = "https://api.open-meteo.com/v1/forecast"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
timezone: str = "Europe/Amsterdam",
|
||||
timeout: int = 10
|
||||
):
|
||||
"""
|
||||
Initialize Open-Meteo client.
|
||||
|
||||
Args:
|
||||
timezone: Default timezone for weather data
|
||||
timeout: HTTP request timeout in seconds
|
||||
"""
|
||||
self.timezone = timezone
|
||||
self.timeout = timeout
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
@property
|
||||
def client(self) -> httpx.AsyncClient:
|
||||
"""Lazy-initialize HTTP client."""
|
||||
if self._client is None or self._client.is_closed:
|
||||
self._client = httpx.AsyncClient(timeout=self.timeout)
|
||||
return self._client
|
||||
|
||||
async def close(self):
|
||||
"""Close HTTP client."""
|
||||
if self._client and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def geocode(self, city: str) -> Optional[GeoLocation]:
|
||||
"""
|
||||
Convert city name to coordinates.
|
||||
|
||||
Args:
|
||||
city: City name (can include country, e.g., "Amsterdam, Netherlands")
|
||||
|
||||
Returns:
|
||||
GeoLocation with coordinates or None if not found
|
||||
"""
|
||||
try:
|
||||
response = await self.client.get(
|
||||
self.GEOCODING_URL,
|
||||
params={
|
||||
"name": city,
|
||||
"count": 1,
|
||||
"language": "en",
|
||||
"format": "json"
|
||||
}
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
results = data.get("results", [])
|
||||
if not results:
|
||||
logger.warning(f"No geocoding results for: {city}")
|
||||
return None
|
||||
|
||||
result = results[0]
|
||||
return GeoLocation(
|
||||
name=result.get("name", city),
|
||||
latitude=result["latitude"],
|
||||
longitude=result["longitude"],
|
||||
country=result.get("country"),
|
||||
admin_area=result.get("admin1") # State/province
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Geocoding request failed for '{city}': {e}")
|
||||
return None
|
||||
except (KeyError, IndexError) as e:
|
||||
logger.error(f"Invalid geocoding response for '{city}': {e}")
|
||||
return None
|
||||
|
||||
async def get_current(self, location: GeoLocation) -> CurrentWeather:
|
||||
"""
|
||||
Get current weather for a location.
|
||||
|
||||
Args:
|
||||
location: GeoLocation with lat/long
|
||||
|
||||
Returns:
|
||||
CurrentWeather with standardized data
|
||||
|
||||
Raises:
|
||||
ValueError: If API request fails
|
||||
"""
|
||||
try:
|
||||
response = await self.client.get(
|
||||
self.WEATHER_URL,
|
||||
params={
|
||||
"latitude": location.latitude,
|
||||
"longitude": location.longitude,
|
||||
"current": [
|
||||
"temperature_2m",
|
||||
"apparent_temperature",
|
||||
"relative_humidity_2m",
|
||||
"weather_code",
|
||||
"wind_speed_10m",
|
||||
"wind_direction_10m"
|
||||
],
|
||||
"timezone": self.timezone,
|
||||
"temperature_unit": "celsius",
|
||||
"wind_speed_unit": "kmh"
|
||||
}
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
current = data.get("current", {})
|
||||
weather_code = current.get("weather_code", 0)
|
||||
|
||||
return CurrentWeather(
|
||||
temperature=current.get("temperature_2m", 0.0),
|
||||
feels_like=current.get("apparent_temperature"),
|
||||
humidity=int(current.get("relative_humidity_2m", 0)),
|
||||
wind_speed=current.get("wind_speed_10m", 0.0),
|
||||
wind_direction=current.get("wind_direction_10m"),
|
||||
condition=WMO_CODE_MAP.get(weather_code, WeatherCondition.UNKNOWN),
|
||||
condition_text=WMO_DESCRIPTIONS.get(weather_code, "Unknown"),
|
||||
timestamp=datetime.now(),
|
||||
location=location.name
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Weather request failed for {location.name}: {e}")
|
||||
raise ValueError(f"Failed to get weather: {e}")
|
||||
|
||||
async def get_forecast(
|
||||
self,
|
||||
location: GeoLocation,
|
||||
days: int = 7
|
||||
) -> WeatherForecast:
|
||||
"""
|
||||
Get weather forecast for a location.
|
||||
|
||||
Args:
|
||||
location: GeoLocation with lat/long
|
||||
days: Number of forecast days (1-16)
|
||||
|
||||
Returns:
|
||||
WeatherForecast with current and daily data
|
||||
|
||||
Raises:
|
||||
ValueError: If API request fails
|
||||
"""
|
||||
days = min(max(days, 1), 16) # Open-Meteo supports 1-16 days
|
||||
|
||||
try:
|
||||
response = await self.client.get(
|
||||
self.WEATHER_URL,
|
||||
params={
|
||||
"latitude": location.latitude,
|
||||
"longitude": location.longitude,
|
||||
"current": [
|
||||
"temperature_2m",
|
||||
"apparent_temperature",
|
||||
"relative_humidity_2m",
|
||||
"weather_code",
|
||||
"wind_speed_10m",
|
||||
"wind_direction_10m"
|
||||
],
|
||||
"daily": [
|
||||
"weather_code",
|
||||
"temperature_2m_max",
|
||||
"temperature_2m_min",
|
||||
"precipitation_sum",
|
||||
"precipitation_probability_max"
|
||||
],
|
||||
"timezone": self.timezone,
|
||||
"temperature_unit": "celsius",
|
||||
"wind_speed_unit": "kmh",
|
||||
"forecast_days": days
|
||||
}
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
# Parse current weather
|
||||
current_data = data.get("current", {})
|
||||
weather_code = current_data.get("weather_code", 0)
|
||||
current = CurrentWeather(
|
||||
temperature=current_data.get("temperature_2m", 0.0),
|
||||
feels_like=current_data.get("apparent_temperature"),
|
||||
humidity=int(current_data.get("relative_humidity_2m", 0)),
|
||||
wind_speed=current_data.get("wind_speed_10m", 0.0),
|
||||
wind_direction=current_data.get("wind_direction_10m"),
|
||||
condition=WMO_CODE_MAP.get(weather_code, WeatherCondition.UNKNOWN),
|
||||
condition_text=WMO_DESCRIPTIONS.get(weather_code, "Unknown"),
|
||||
timestamp=datetime.now(),
|
||||
location=location.name
|
||||
)
|
||||
|
||||
# Parse daily forecast
|
||||
daily_data = data.get("daily", {})
|
||||
daily = []
|
||||
dates = daily_data.get("time", [])
|
||||
for i, date_str in enumerate(dates):
|
||||
code = daily_data.get("weather_code", [])[i] if i < len(daily_data.get("weather_code", [])) else 0
|
||||
daily.append(DayForecast(
|
||||
date=datetime.fromisoformat(date_str),
|
||||
temp_high=daily_data.get("temperature_2m_max", [])[i] if i < len(daily_data.get("temperature_2m_max", [])) else 0.0,
|
||||
temp_low=daily_data.get("temperature_2m_min", [])[i] if i < len(daily_data.get("temperature_2m_min", [])) else 0.0,
|
||||
condition=WMO_CODE_MAP.get(code, WeatherCondition.UNKNOWN),
|
||||
condition_text=WMO_DESCRIPTIONS.get(code, "Unknown"),
|
||||
precipitation_chance=daily_data.get("precipitation_probability_max", [])[i] if i < len(daily_data.get("precipitation_probability_max", [])) else None,
|
||||
precipitation_mm=daily_data.get("precipitation_sum", [])[i] if i < len(daily_data.get("precipitation_sum", [])) else None
|
||||
))
|
||||
|
||||
return WeatherForecast(
|
||||
location=location.name,
|
||||
current=current,
|
||||
daily=daily
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Forecast request failed for {location.name}: {e}")
|
||||
raise ValueError(f"Failed to get forecast: {e}")
|
||||
@@ -0,0 +1,217 @@
|
||||
"""
|
||||
Client for central Tatlock settings database.
|
||||
|
||||
Reads settings from the shared system_settings PostgreSQL database.
|
||||
Writes are done via psql CLI or future CRUD manager.
|
||||
"""
|
||||
|
||||
import asyncpg
|
||||
import logging
|
||||
from typing import Optional, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SettingsClient:
|
||||
"""Client for system_settings database."""
|
||||
|
||||
def __init__(self, dsn: str):
|
||||
"""
|
||||
Initialize settings client.
|
||||
|
||||
Args:
|
||||
dsn: PostgreSQL connection string
|
||||
e.g., "postgresql://settings:password@postgres-shared:5432/system_settings"
|
||||
"""
|
||||
self.dsn = dsn
|
||||
self._pool: Optional[asyncpg.Pool] = None
|
||||
|
||||
async def connect(self):
|
||||
"""Initialize connection pool."""
|
||||
if not self._pool:
|
||||
try:
|
||||
self._pool = await asyncpg.create_pool(
|
||||
self.dsn,
|
||||
min_size=1,
|
||||
max_size=5,
|
||||
command_timeout=10,
|
||||
)
|
||||
logger.info("Connected to system_settings database")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to system_settings: {e}")
|
||||
raise
|
||||
|
||||
async def close(self):
|
||||
"""Close connection pool."""
|
||||
if self._pool:
|
||||
await self._pool.close()
|
||||
self._pool = None
|
||||
logger.info("Disconnected from system_settings database")
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
"""Check database connectivity."""
|
||||
try:
|
||||
await self.connect()
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.fetchval("SELECT 1")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Settings database health check failed: {e}")
|
||||
return False
|
||||
|
||||
async def get(self, key: str, user_scope: str = "global") -> Optional[Any]:
|
||||
"""
|
||||
Get a setting by key with user fallback to global.
|
||||
|
||||
Args:
|
||||
key: Setting key (e.g., "api.openmeteo", "weather.units")
|
||||
user_scope: User identifier or "global"
|
||||
|
||||
Returns:
|
||||
Setting value (parsed from JSONB) or None if not found.
|
||||
User-specific value takes precedence over global.
|
||||
"""
|
||||
await self.connect()
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT value FROM settings
|
||||
WHERE key = $1 AND user_scope IN ($2, 'global')
|
||||
ORDER BY CASE WHEN user_scope = $2 THEN 0 ELSE 1 END
|
||||
LIMIT 1
|
||||
""",
|
||||
key, user_scope
|
||||
)
|
||||
if row:
|
||||
return row["value"]
|
||||
return None
|
||||
|
||||
async def get_with_schema(self, key: str, user_scope: str = "global") -> Optional[dict]:
|
||||
"""
|
||||
Get a setting with its JSON Schema.
|
||||
|
||||
Returns:
|
||||
Dict with "value" and "schema" keys, or None if not found.
|
||||
"""
|
||||
await self.connect()
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT value, schema FROM settings
|
||||
WHERE key = $1 AND user_scope IN ($2, 'global')
|
||||
ORDER BY CASE WHEN user_scope = $2 THEN 0 ELSE 1 END
|
||||
LIMIT 1
|
||||
""",
|
||||
key, user_scope
|
||||
)
|
||||
if row:
|
||||
return {"value": row["value"], "schema": row["schema"]}
|
||||
return None
|
||||
|
||||
async def get_by_prefix(self, prefix: str, user_scope: str = "global") -> dict[str, Any]:
|
||||
"""
|
||||
Get all settings matching a key prefix.
|
||||
|
||||
Args:
|
||||
prefix: Key prefix (e.g., "api." for all API configs)
|
||||
user_scope: User identifier or "global"
|
||||
|
||||
Returns:
|
||||
Dict mapping keys to values. User-specific values override global.
|
||||
"""
|
||||
await self.connect()
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT DISTINCT ON (key) key, value FROM settings
|
||||
WHERE key LIKE $1 AND user_scope IN ($2, 'global')
|
||||
ORDER BY key, CASE WHEN user_scope = $2 THEN 0 ELSE 1 END
|
||||
""",
|
||||
f"{prefix}%", user_scope
|
||||
)
|
||||
return {row["key"]: row["value"] for row in rows}
|
||||
|
||||
async def get_api_config(self, service: str) -> Optional[dict]:
|
||||
"""
|
||||
Get API configuration for a service.
|
||||
|
||||
Args:
|
||||
service: Service name (e.g., "openmeteo", "nos", "alphavantage")
|
||||
|
||||
Returns:
|
||||
API config dict or None if not found.
|
||||
"""
|
||||
value = await self.get(f"api.{service}")
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return None
|
||||
|
||||
async def get_api_key(self, service: str) -> Optional[str]:
|
||||
"""
|
||||
Get API key for a service if enabled.
|
||||
|
||||
Args:
|
||||
service: Service name (e.g., "alphavantage")
|
||||
|
||||
Returns:
|
||||
API key string or None if not found or disabled.
|
||||
"""
|
||||
config = await self.get_api_config(service)
|
||||
if config:
|
||||
# Check if explicitly disabled
|
||||
if config.get("enabled") is False:
|
||||
return None
|
||||
return config.get("api_key")
|
||||
return None
|
||||
|
||||
async def is_api_enabled(self, service: str) -> bool:
|
||||
"""
|
||||
Check if an API service is enabled.
|
||||
|
||||
Args:
|
||||
service: Service name (e.g., "alphavantage", "openmeteo")
|
||||
|
||||
Returns:
|
||||
True if enabled (or no explicit setting), False if disabled.
|
||||
"""
|
||||
config = await self.get_api_config(service)
|
||||
if config:
|
||||
# Default to enabled if not specified
|
||||
return config.get("enabled", True)
|
||||
return False # No config means not available
|
||||
|
||||
async def get_user_preference(self, key: str, user: str) -> Optional[Any]:
|
||||
"""
|
||||
Get a user-specific preference.
|
||||
|
||||
Args:
|
||||
key: Preference key (e.g., "weather.units", "news.sources")
|
||||
user: User identifier
|
||||
|
||||
Returns:
|
||||
Preference value or None if not set.
|
||||
"""
|
||||
return await self.get(key, user_scope=user)
|
||||
|
||||
async def list_keys(self, user_scope: Optional[str] = None) -> list[str]:
|
||||
"""
|
||||
List all setting keys, optionally filtered by user_scope.
|
||||
|
||||
Args:
|
||||
user_scope: Filter by scope (None for all)
|
||||
|
||||
Returns:
|
||||
List of setting keys.
|
||||
"""
|
||||
await self.connect()
|
||||
async with self._pool.acquire() as conn:
|
||||
if user_scope:
|
||||
rows = await conn.fetch(
|
||||
"SELECT key FROM settings WHERE user_scope = $1 ORDER BY key",
|
||||
user_scope
|
||||
)
|
||||
else:
|
||||
rows = await conn.fetch(
|
||||
"SELECT DISTINCT key FROM settings ORDER BY key"
|
||||
)
|
||||
return [row["key"] for row in rows]
|
||||
@@ -121,6 +121,13 @@ class Settings(BaseSettings):
|
||||
maintenance_orphan_cleanup_enabled: bool = Field(default=True, description="Enable automatic orphan cleanup")
|
||||
maintenance_cleanup_batch_size: int = Field(default=100, ge=10, le=1000, description="Cleanup batch size")
|
||||
|
||||
# Central Settings Database (Tatlock-wide)
|
||||
system_settings_host: str = Field(default="postgres-shared", description="System settings PostgreSQL host")
|
||||
system_settings_port: int = Field(default=5432, description="System settings PostgreSQL port")
|
||||
system_settings_db: str = Field(default="system_settings", description="System settings database name")
|
||||
system_settings_user: str = Field(default="settings", description="System settings database user")
|
||||
system_settings_password: str = Field(default="", description="System settings database password")
|
||||
|
||||
@property
|
||||
def qdrant_url(self) -> str:
|
||||
"""Computed Qdrant URL."""
|
||||
@@ -131,6 +138,16 @@ class Settings(BaseSettings):
|
||||
"""Computed Redis URL."""
|
||||
return f"redis://{self.redis_host}:{self.redis_port}/{self.redis_db}"
|
||||
|
||||
@property
|
||||
def system_settings_dsn(self) -> str:
|
||||
"""Computed System Settings PostgreSQL DSN."""
|
||||
if not self.system_settings_password:
|
||||
return ""
|
||||
return (
|
||||
f"postgresql://{self.system_settings_user}:{self.system_settings_password}"
|
||||
f"@{self.system_settings_host}:{self.system_settings_port}/{self.system_settings_db}"
|
||||
)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
|
||||
@@ -23,6 +23,12 @@ from src.clients.searxng_client import SearXNGClient
|
||||
from src.clients.ollama_client import OllamaClient
|
||||
from src.clients.content_extractor import ContentExtractor
|
||||
from src.clients.paperless_client import PaperlessClient
|
||||
from src.clients.settings_client import SettingsClient
|
||||
from src.apis import (
|
||||
OpenMeteoProvider,
|
||||
AggregatedNewsProvider,
|
||||
AlphaVantageProvider,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -178,6 +184,131 @@ def get_paperless_client() -> PaperlessClient:
|
||||
return client
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings_client() -> SettingsClient:
|
||||
"""
|
||||
Get central settings database client singleton.
|
||||
|
||||
Returns:
|
||||
Initialized SettingsClient for Tatlock system_settings database
|
||||
|
||||
Note: Returns client with empty DSN if password not configured
|
||||
"""
|
||||
settings = get_settings()
|
||||
if not settings.system_settings_password:
|
||||
logger.warning("System settings password not configured - settings database disabled")
|
||||
client = SettingsClient(dsn=settings.system_settings_dsn)
|
||||
logger.debug(f"Created Settings client: {settings.system_settings_host}")
|
||||
return client
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# External API Providers
|
||||
# =============================================================================
|
||||
|
||||
@lru_cache
|
||||
def get_weather_provider() -> OpenMeteoProvider:
|
||||
"""
|
||||
Get Open-Meteo weather provider singleton.
|
||||
|
||||
Returns:
|
||||
Initialized OpenMeteoProvider with default timezone
|
||||
|
||||
Note: Timezone can be overridden per-request for user preferences
|
||||
"""
|
||||
provider = OpenMeteoProvider(timezone="Europe/Amsterdam")
|
||||
logger.debug("Created OpenMeteo weather provider")
|
||||
return provider
|
||||
|
||||
|
||||
# News provider requires sources from settings database
|
||||
_news_provider: AggregatedNewsProvider | None = None
|
||||
|
||||
|
||||
async def get_news_provider() -> AggregatedNewsProvider:
|
||||
"""
|
||||
Get aggregated news provider.
|
||||
|
||||
Returns:
|
||||
Initialized AggregatedNewsProvider with user-configured sources
|
||||
and per-source category filters.
|
||||
|
||||
Note: Configuration is fetched from system_settings database:
|
||||
- news.sources: list of enabled sources (default: ["nos", "bbc"])
|
||||
- api.{source}.categories: list of enabled categories per source
|
||||
"""
|
||||
global _news_provider
|
||||
if _news_provider is not None:
|
||||
return _news_provider
|
||||
|
||||
settings_client = get_settings_client()
|
||||
|
||||
# Get enabled sources
|
||||
sources = await settings_client.get("news.sources")
|
||||
if not sources or not isinstance(sources, list):
|
||||
sources = ["nos", "bbc"]
|
||||
logger.info(f"Using default news sources: {sources}")
|
||||
else:
|
||||
logger.info(f"Using configured news sources: {sources}")
|
||||
|
||||
# Filter out disabled sources and get category filters
|
||||
enabled_sources: list[str] = []
|
||||
category_filters: dict[str, list[str]] = {}
|
||||
|
||||
for source in sources:
|
||||
config = await settings_client.get_api_config(source)
|
||||
if config:
|
||||
# Check if source is disabled
|
||||
if config.get("enabled") is False:
|
||||
logger.info(f"News source '{source}' is disabled - skipping")
|
||||
continue
|
||||
# Get category filter if specified
|
||||
categories = config.get("categories", [])
|
||||
if categories:
|
||||
category_filters[source] = categories
|
||||
logger.debug(f"Source '{source}' categories: {categories}")
|
||||
enabled_sources.append(source)
|
||||
|
||||
if not enabled_sources:
|
||||
enabled_sources = ["nos", "bbc"]
|
||||
logger.warning("No enabled news sources - using defaults")
|
||||
|
||||
_news_provider = AggregatedNewsProvider(
|
||||
sources=enabled_sources,
|
||||
category_filters=category_filters
|
||||
)
|
||||
return _news_provider
|
||||
|
||||
|
||||
# AlphaVantage requires API key from settings database
|
||||
_alphavantage_provider: AlphaVantageProvider | None = None
|
||||
|
||||
|
||||
async def get_alphavantage_provider() -> AlphaVantageProvider | None:
|
||||
"""
|
||||
Get Alpha Vantage financial provider.
|
||||
|
||||
Returns:
|
||||
Initialized AlphaVantageProvider or None if API key not configured
|
||||
|
||||
Note: API key is fetched from system_settings database
|
||||
"""
|
||||
global _alphavantage_provider
|
||||
if _alphavantage_provider is not None:
|
||||
return _alphavantage_provider
|
||||
|
||||
settings_client = get_settings_client()
|
||||
api_key = await settings_client.get_api_key("alphavantage")
|
||||
|
||||
if not api_key:
|
||||
logger.warning("Alpha Vantage API key not configured - financial provider disabled")
|
||||
return None
|
||||
|
||||
_alphavantage_provider = AlphaVantageProvider(api_key=api_key)
|
||||
logger.debug("Created Alpha Vantage financial provider")
|
||||
return _alphavantage_provider
|
||||
|
||||
|
||||
# Type aliases for FastAPI endpoint dependencies
|
||||
# Usage: def my_endpoint(neo4j: Neo4jDep):
|
||||
Neo4jDep = Annotated[Neo4jClient, Depends(get_neo4j_client)]
|
||||
@@ -188,6 +319,12 @@ OllamaDep = Annotated[OllamaClient, Depends(get_ollama_client)]
|
||||
RedisDep = Annotated[aioredis.Redis, Depends(get_redis_client)]
|
||||
ContentExtractorDep = Annotated[ContentExtractor, Depends(get_content_extractor)]
|
||||
PaperlessDep = Annotated[PaperlessClient, Depends(get_paperless_client)]
|
||||
SettingsClientDep = Annotated[SettingsClient, Depends(get_settings_client)]
|
||||
|
||||
# External API provider dependencies
|
||||
WeatherProviderDep = Annotated[OpenMeteoProvider, Depends(get_weather_provider)]
|
||||
NewsProviderDep = Annotated[AggregatedNewsProvider, Depends(get_news_provider)]
|
||||
AlphaVantageProviderDep = Annotated[AlphaVantageProvider | None, Depends(get_alphavantage_provider)]
|
||||
|
||||
|
||||
# Lifecycle management functions
|
||||
@@ -241,6 +378,20 @@ async def startup_clients():
|
||||
else:
|
||||
logger.info("○ Paperless-ngx not configured (document storage disabled)")
|
||||
|
||||
# Check System Settings database availability
|
||||
if settings.system_settings_password:
|
||||
try:
|
||||
settings_client = get_settings_client()
|
||||
is_healthy = await settings_client.health_check()
|
||||
if is_healthy:
|
||||
logger.info(f"✓ System settings DB ready: {settings.system_settings_host}")
|
||||
else:
|
||||
logger.warning("✗ System settings DB not responding")
|
||||
except Exception as e:
|
||||
logger.error(f"✗ System settings health check failed: {e}")
|
||||
else:
|
||||
logger.info("○ System settings not configured")
|
||||
|
||||
# Qdrant, Wiki.js, SearXNG are lazy-initialized
|
||||
logger.info("Service clients startup complete")
|
||||
|
||||
@@ -271,6 +422,7 @@ async def shutdown_clients():
|
||||
("SearXNG", get_searxng_client()),
|
||||
("Ollama", get_ollama_client()),
|
||||
("Paperless", get_paperless_client()),
|
||||
("OpenMeteo", get_weather_provider()),
|
||||
]
|
||||
|
||||
for name, client in clients_to_close:
|
||||
@@ -280,6 +432,35 @@ async def shutdown_clients():
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing {name} client: {e}")
|
||||
|
||||
# Close async-initialized providers
|
||||
global _news_provider, _alphavantage_provider
|
||||
|
||||
if _news_provider is not None:
|
||||
try:
|
||||
await _news_provider.close()
|
||||
_news_provider = None
|
||||
logger.info("✓ News provider closed")
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing News provider: {e}")
|
||||
|
||||
if _alphavantage_provider is not None:
|
||||
try:
|
||||
await _alphavantage_provider.close()
|
||||
_alphavantage_provider = None
|
||||
logger.info("✓ AlphaVantage client closed")
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing AlphaVantage client: {e}")
|
||||
|
||||
# Close settings database connection
|
||||
settings = get_settings()
|
||||
if settings.system_settings_password:
|
||||
try:
|
||||
settings_client = get_settings_client()
|
||||
await settings_client.close()
|
||||
logger.info("✓ System settings client closed")
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing settings client: {e}")
|
||||
|
||||
logger.info("Service clients shutdown complete")
|
||||
|
||||
|
||||
@@ -364,6 +545,17 @@ async def check_service_health() -> dict:
|
||||
else:
|
||||
health["paperless"] = None # Not configured
|
||||
|
||||
# System Settings database
|
||||
if settings.system_settings_password:
|
||||
try:
|
||||
settings_client = get_settings_client()
|
||||
health["system_settings"] = await settings_client.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"System settings health check failed: {e}")
|
||||
health["system_settings"] = False
|
||||
else:
|
||||
health["system_settings"] = None # Not configured
|
||||
|
||||
return health
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user