33 lines
791 B
Python
33 lines
791 B
Python
"""
|
|
Configuration for web scraper module
|
|
"""
|
|
from pydantic_settings import BaseSettings
|
|
from functools import lru_cache
|
|
|
|
|
|
class WebScraperSettings(BaseSettings):
|
|
"""Web scraper specific settings"""
|
|
|
|
# HTTP client configuration
|
|
request_timeout: int = 30
|
|
max_redirects: int = 5
|
|
user_agent: str = "Mozilla/5.0 (compatible; CoreCode/1.0)"
|
|
|
|
# Content extraction
|
|
default_max_length: int = 10000
|
|
max_links_to_extract: int = 50
|
|
|
|
# Rate limiting (future use)
|
|
rate_limit_enabled: bool = False
|
|
requests_per_minute: int = 60
|
|
|
|
class Config:
|
|
env_prefix = "WEB_SCRAPER_"
|
|
case_sensitive = False
|
|
|
|
|
|
@lru_cache()
|
|
def get_web_scraper_settings() -> WebScraperSettings:
|
|
"""Cached web scraper settings instance"""
|
|
return WebScraperSettings()
|