Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4cfad2e8bc | ||
|
|
1e31e74ad6 | ||
|
|
72f515bf61 | ||
|
|
0c085d603e |
@@ -1,12 +1,25 @@
|
||||
name: Build and Push
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create Gitea Release
|
||||
run: |
|
||||
curl -sf -X POST \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"tag_name": "${{ github.ref_name }}", "name": "Release ${{ github.ref_name }}", "body": "Automated release for ${{ github.ref_name }}"}' \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases"
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
needs: release
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
|
||||
@@ -5,6 +5,32 @@ All notable changes to Library Desk will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [1.7.2] - 2026-01-07
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Volatile TTL doubled** - TTL now 2x refresh interval to survive missed/delayed scheduler runs
|
||||
|
||||
## [1.7.1] - 2026-01-07
|
||||
|
||||
### Fixed
|
||||
|
||||
- **CI workflow** - Updated Gitea Actions to trigger on tag push (matching core-api)
|
||||
|
||||
## [1.7.0] - 2026-01-07
|
||||
|
||||
### Added
|
||||
|
||||
- **Combined Environment Endpoint** - `POST /volatile/fetch/environment/{city}`
|
||||
- Fetches weather and air quality concurrently with `asyncio.gather()`
|
||||
- Single geocode lookup shared between both API calls
|
||||
- More efficient than calling weather and air_quality separately
|
||||
- Reduces wall-clock time and eliminates redundant geocoding
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Scheduler executor name** - Fixed `rest_api` → `rest_api_executor` in SchedulerTask model and register_volatile_fetch() to prevent "Executor module not found" errors
|
||||
|
||||
## [1.6.2] - 2025-12-30
|
||||
|
||||
### Added
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quick script to check environmental data in Qdrant volatile cache."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
async def main():
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.models import Filter, FieldCondition, MatchValue
|
||||
from src.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
qdrant = QdrantClient(url=settings.qdrant_url)
|
||||
|
||||
user = "jpmschweitzer"
|
||||
collection = f"volatile_{user}"
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Volatile Data in Qdrant ({collection})")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
namespaces = ["weather", "air_quality", "forecast", "sun", "news"]
|
||||
|
||||
for ns in namespaces:
|
||||
try:
|
||||
results = qdrant.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=Filter(
|
||||
must=[FieldCondition(key="namespace", match=MatchValue(value=ns))]
|
||||
),
|
||||
limit=10,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
|
||||
points = results[0]
|
||||
print(f"=== {ns.upper()} ({len(points)} records) ===")
|
||||
|
||||
if not points:
|
||||
print(" (no data)")
|
||||
print()
|
||||
continue
|
||||
|
||||
for point in points:
|
||||
payload = point.payload
|
||||
raw_data = payload.get("raw_data", {})
|
||||
|
||||
if ns == "weather":
|
||||
print(f" Temperature: {raw_data.get('temperature')}°C (feels like {raw_data.get('feels_like')}°C)")
|
||||
print(f" Conditions: {raw_data.get('conditions')}")
|
||||
print(f" Humidity: {raw_data.get('humidity')}%")
|
||||
print(f" Wind: {raw_data.get('wind_speed')} km/h")
|
||||
print(f" UV Index: {raw_data.get('uv_index')}")
|
||||
|
||||
elif ns == "air_quality":
|
||||
print(f" European AQI: {raw_data.get('aqi_european')}")
|
||||
print(f" US AQI: {raw_data.get('aqi_us')}")
|
||||
print(f" PM2.5: {raw_data.get('pm2_5')} µg/m³")
|
||||
print(f" PM10: {raw_data.get('pm10')} µg/m³")
|
||||
print(f" Ozone: {raw_data.get('ozone')} µg/m³")
|
||||
print(f" NO₂: {raw_data.get('nitrogen_dioxide')} µg/m³")
|
||||
|
||||
elif ns == "forecast":
|
||||
daily = raw_data.get("daily", [])
|
||||
for day in daily[:5]:
|
||||
print(f" {day.get('day_name', 'N/A')[:3]}: {day.get('temp_low'):.0f}-{day.get('temp_high'):.0f}°C, {day.get('conditions')}")
|
||||
|
||||
elif ns == "sun":
|
||||
print(f" Sunrise: {raw_data.get('sunrise')}")
|
||||
print(f" Sunset: {raw_data.get('sunset')}")
|
||||
print(f" Daylight: {raw_data.get('daylight_hours', 0):.1f} hours")
|
||||
|
||||
elif ns == "news":
|
||||
headlines = raw_data.get("headlines", [])
|
||||
print(f" Category: {raw_data.get('category', 'general')}")
|
||||
print(f" Headlines ({len(headlines)}):")
|
||||
for item in headlines[:5]:
|
||||
title = item.get("title", "")[:60]
|
||||
source = item.get("source", "")
|
||||
print(f" - [{source}] {title}...")
|
||||
|
||||
# Show TTL info
|
||||
ttl_expiry = payload.get("ttl_expiry")
|
||||
if ttl_expiry:
|
||||
remaining = (ttl_expiry / 1000) - time.time()
|
||||
if remaining > 0:
|
||||
print(f" TTL remaining: {int(remaining)}s ({int(remaining/60)} min)")
|
||||
else:
|
||||
print(f" TTL: EXPIRED")
|
||||
|
||||
print()
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error fetching {ns}: {e}")
|
||||
print()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "library-desk"
|
||||
version = "1.6.2"
|
||||
version = "1.7.2"
|
||||
description = "Coordination service for The Library system - HybridRAG queries, document ingestion, entity extraction, and knowledge consolidation"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
|
||||
@@ -18,7 +18,7 @@ class SchedulerTask(BaseModel):
|
||||
|
||||
task_name: str = Field(..., description="Unique task identifier")
|
||||
service: str = Field(default="library-desk", description="Service that owns this task")
|
||||
executor: str = Field(default="rest_api", description="Executor type")
|
||||
executor: str = Field(default="rest_api_executor", description="Executor type")
|
||||
priority: int = Field(default=50, ge=1, le=100, description="Priority (lower = higher)")
|
||||
description: Optional[str] = Field(None, description="Human-readable description")
|
||||
enabled: bool = Field(default=True, description="Whether task is enabled")
|
||||
@@ -291,7 +291,7 @@ class SchedulerClient:
|
||||
task = SchedulerTask(
|
||||
task_name=task_name,
|
||||
service="library-desk",
|
||||
executor="rest_api",
|
||||
executor="rest_api_executor",
|
||||
priority=60, # Background maintenance priority
|
||||
description=description or f"Prefetch {namespace}/{key} for {user}",
|
||||
minute=schedule.get("minute", -1),
|
||||
|
||||
+14
-13
@@ -38,20 +38,21 @@ class VolatileNamespace(str, Enum):
|
||||
|
||||
|
||||
# Default TTLs per namespace (in seconds)
|
||||
# TTL = 2x refresh interval to ensure data survives missed/delayed refreshes
|
||||
NAMESPACE_DEFAULT_TTL: Dict[str, int] = {
|
||||
VolatileNamespace.WEATHER: 3600, # 1 hour - current conditions
|
||||
VolatileNamespace.FORECAST: 43200, # 12 hours - forecast stable longer
|
||||
VolatileNamespace.SUN: 86400, # 24 hours - sun times change daily
|
||||
VolatileNamespace.NEWS: 3600, # 1 hour - news cycles
|
||||
VolatileNamespace.FINANCIAL: 300, # 5 min - markets move fast
|
||||
VolatileNamespace.TRANSIT: 300, # 5 min - schedules update frequently
|
||||
VolatileNamespace.TRAFFIC: 600, # 10 min - traffic patterns
|
||||
VolatileNamespace.AIR_QUALITY: 3600, # 1 hour - air quality stable
|
||||
VolatileNamespace.SPORTS: 60, # 1 min - live scores
|
||||
VolatileNamespace.SOCIAL: 600, # 10 min - social notifications
|
||||
VolatileNamespace.SYSTEM: 60, # 1 min - system health
|
||||
VolatileNamespace.CONTEXT: 3600, # 1 hour - session context
|
||||
VolatileNamespace.CUSTOM: 3600, # 1 hour - default for custom
|
||||
VolatileNamespace.WEATHER: 7200, # 2 hours (hourly refresh)
|
||||
VolatileNamespace.FORECAST: 86400, # 24 hours (12hr refresh)
|
||||
VolatileNamespace.SUN: 172800, # 48 hours (daily refresh)
|
||||
VolatileNamespace.NEWS: 7200, # 2 hours (hourly refresh)
|
||||
VolatileNamespace.FINANCIAL: 600, # 10 min (5 min refresh)
|
||||
VolatileNamespace.TRANSIT: 600, # 10 min (5 min refresh)
|
||||
VolatileNamespace.TRAFFIC: 1200, # 20 min (10 min refresh)
|
||||
VolatileNamespace.AIR_QUALITY: 7200, # 2 hours (hourly refresh)
|
||||
VolatileNamespace.SPORTS: 120, # 2 min (1 min refresh)
|
||||
VolatileNamespace.SOCIAL: 1200, # 20 min (10 min refresh)
|
||||
VolatileNamespace.SYSTEM: 120, # 2 min (1 min refresh)
|
||||
VolatileNamespace.CONTEXT: 7200, # 2 hours - session context
|
||||
VolatileNamespace.CUSTOM: 7200, # 2 hours - default for custom
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -545,6 +545,64 @@ async def fetch_air_quality(
|
||||
}
|
||||
|
||||
|
||||
@router.post("/fetch/environment/{city}")
|
||||
async def fetch_environment(
|
||||
city: str,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
weather_ttl: int = Query(default=3600, ge=60, le=86400, description="Weather TTL in seconds"),
|
||||
air_quality_ttl: int = Query(default=3600, ge=60, le=86400, description="Air quality TTL in seconds"),
|
||||
qdrant: QdrantDep = None,
|
||||
ollama: OllamaDep = None,
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Fetch weather and air quality concurrently for a city.
|
||||
|
||||
Performs a single geocode lookup and fetches both weather and air quality
|
||||
data in parallel, storing both in volatile cache. More efficient than
|
||||
calling /fetch/weather and /fetch/air_quality separately.
|
||||
|
||||
**Example:**
|
||||
```
|
||||
POST /volatile/fetch/environment/rotterdam?user=jpmschweitzer
|
||||
```
|
||||
|
||||
**Response includes:**
|
||||
- weather: Current conditions (temperature, humidity, wind, UV)
|
||||
- air_quality: AQI indices, pollutants, pollen data
|
||||
"""
|
||||
volatile_service = get_volatile_service(qdrant, ollama)
|
||||
weather_provider = get_weather_provider()
|
||||
|
||||
fetch_service = VolatileFetchService(
|
||||
volatile_service=volatile_service,
|
||||
weather_provider=weather_provider,
|
||||
)
|
||||
|
||||
result = await fetch_service.fetch_environment(
|
||||
user, city, weather_ttl=weather_ttl, air_quality_ttl=air_quality_ttl
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
raise HTTPException(status_code=500, detail="; ".join(result.errors))
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"key": result.key,
|
||||
"weather": {
|
||||
"success": result.weather.success if result.weather else False,
|
||||
"record": result.weather.record if result.weather else None,
|
||||
"error": result.weather.error if result.weather else None,
|
||||
},
|
||||
"air_quality": {
|
||||
"success": result.air_quality.success if result.air_quality else False,
|
||||
"record": result.air_quality.record if result.air_quality else None,
|
||||
"error": result.air_quality.error if result.air_quality else None,
|
||||
},
|
||||
"errors": result.errors,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{namespace}/{key}", response_model=VolatileRecordResponse)
|
||||
async def get_record(
|
||||
namespace: str,
|
||||
|
||||
@@ -5,9 +5,10 @@ Orchestrates fetching data from external APIs and storing in volatile cache.
|
||||
Called by scheduler for prefetch or by HybridRAG for reactive caching.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Optional
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from src.apis import (
|
||||
OpenMeteoProvider,
|
||||
@@ -36,6 +37,16 @@ class FetchResult:
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnvironmentFetchResult:
|
||||
"""Result of combined environment fetch (weather + air quality)."""
|
||||
success: bool
|
||||
key: str
|
||||
weather: Optional[FetchResult] = None
|
||||
air_quality: Optional[FetchResult] = None
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class VolatileFetchService:
|
||||
"""
|
||||
Service to fetch external data and store in volatile cache.
|
||||
@@ -596,3 +607,130 @@ class VolatileFetchService:
|
||||
key=city.lower(),
|
||||
error=str(e)
|
||||
)
|
||||
|
||||
async def fetch_environment(
|
||||
self,
|
||||
user: str,
|
||||
city: str,
|
||||
weather_ttl: int = 3600,
|
||||
air_quality_ttl: int = 3600,
|
||||
) -> EnvironmentFetchResult:
|
||||
"""
|
||||
Fetch weather and air quality concurrently for a city.
|
||||
|
||||
Performs a single geocode lookup and fetches both weather and air quality
|
||||
data in parallel, storing both in volatile cache.
|
||||
|
||||
Args:
|
||||
user: User identifier
|
||||
city: City name (will be geocoded once)
|
||||
weather_ttl: TTL for weather data (default 1 hour)
|
||||
air_quality_ttl: TTL for air quality data (default 1 hour)
|
||||
|
||||
Returns:
|
||||
EnvironmentFetchResult with both weather and air quality results
|
||||
"""
|
||||
errors: list[str] = []
|
||||
key = city.lower()
|
||||
|
||||
# Single geocode lookup (shared by both fetches)
|
||||
try:
|
||||
location = await self.weather.geocode(city)
|
||||
if not location:
|
||||
return EnvironmentFetchResult(
|
||||
success=False,
|
||||
key=key,
|
||||
errors=[f"Could not geocode city: {city}"]
|
||||
)
|
||||
except Exception as e:
|
||||
return EnvironmentFetchResult(
|
||||
success=False,
|
||||
key=key,
|
||||
errors=[f"Geocoding failed: {e}"]
|
||||
)
|
||||
|
||||
# Fetch weather and air quality concurrently
|
||||
async def fetch_weather_data() -> FetchResult:
|
||||
try:
|
||||
current = await self.weather.get_current(location)
|
||||
text = current.to_text()
|
||||
data = {
|
||||
"temperature": current.temperature,
|
||||
"feels_like": current.feels_like,
|
||||
"humidity": current.humidity,
|
||||
"wind_speed": current.wind_speed,
|
||||
"wind_direction": current.wind_direction,
|
||||
"conditions": current.condition_text,
|
||||
"condition_code": current.condition.value,
|
||||
"uv_index": current.uv_index,
|
||||
"location": current.location,
|
||||
"text": text,
|
||||
}
|
||||
record = await self.volatile.store(
|
||||
user=user,
|
||||
namespace=VolatileNamespace.WEATHER,
|
||||
key=key,
|
||||
data=data,
|
||||
source="openmeteo",
|
||||
ttl=weather_ttl,
|
||||
)
|
||||
return FetchResult(success=True, namespace="weather", key=key, record=record)
|
||||
except Exception as e:
|
||||
return FetchResult(success=False, namespace="weather", key=key, error=str(e))
|
||||
|
||||
async def fetch_air_quality_data() -> FetchResult:
|
||||
try:
|
||||
air_quality = await self.weather.get_air_quality(location)
|
||||
data = {
|
||||
"location": air_quality.location,
|
||||
"aqi_european": air_quality.aqi_european,
|
||||
"aqi_us": air_quality.aqi_us,
|
||||
"pm2_5": air_quality.pm2_5,
|
||||
"pm10": air_quality.pm10,
|
||||
"ozone": air_quality.ozone,
|
||||
"nitrogen_dioxide": air_quality.nitrogen_dioxide,
|
||||
"sulphur_dioxide": air_quality.sulphur_dioxide,
|
||||
"carbon_monoxide": air_quality.carbon_monoxide,
|
||||
"pollen_grass": air_quality.pollen_grass,
|
||||
"pollen_birch": air_quality.pollen_birch,
|
||||
"pollen_alder": air_quality.pollen_alder,
|
||||
"text": air_quality.to_text(),
|
||||
}
|
||||
record = await self.volatile.store(
|
||||
user=user,
|
||||
namespace=VolatileNamespace.AIR_QUALITY,
|
||||
key=key,
|
||||
data=data,
|
||||
source="openmeteo",
|
||||
ttl=air_quality_ttl,
|
||||
)
|
||||
return FetchResult(success=True, namespace="air_quality", key=key, record=record)
|
||||
except Exception as e:
|
||||
return FetchResult(success=False, namespace="air_quality", key=key, error=str(e))
|
||||
|
||||
# Run both fetches concurrently
|
||||
weather_result, air_quality_result = await asyncio.gather(
|
||||
fetch_weather_data(),
|
||||
fetch_air_quality_data(),
|
||||
)
|
||||
|
||||
# Collect any errors
|
||||
if not weather_result.success:
|
||||
errors.append(f"Weather: {weather_result.error}")
|
||||
if not air_quality_result.success:
|
||||
errors.append(f"Air quality: {air_quality_result.error}")
|
||||
|
||||
success = weather_result.success or air_quality_result.success
|
||||
logger.info(
|
||||
f"Environment fetch for {city} (user={user}): "
|
||||
f"weather={'ok' if weather_result.success else 'failed'}, "
|
||||
f"air_quality={'ok' if air_quality_result.success else 'failed'}"
|
||||
)
|
||||
|
||||
return EnvironmentFetchResult(
|
||||
success=success,
|
||||
key=key,
|
||||
weather=weather_result,
|
||||
air_quality=air_quality_result,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user