diff --git a/src/controllers/health_controller.py b/src/controllers/health_controller.py index 9a237da..5491b46 100644 --- a/src/controllers/health_controller.py +++ b/src/controllers/health_controller.py @@ -61,7 +61,7 @@ class HealthController(BaseController): "chat_completions": "/v1/chat/completions", "models": "/v1/models", "conversations": "/v1/conversations", - "web_scraper": "/web-scraper/scrape", + "dns_lookup": "/tools/dns/lookup", "infrastructure": "/infrastructure", "health": "/health", "health_full": "/health/full" diff --git a/src/controllers/tools_controller.py b/src/controllers/tools_controller.py index 7313772..d908880 100644 --- a/src/controllers/tools_controller.py +++ b/src/controllers/tools_controller.py @@ -2,16 +2,12 @@ Tools Controller Provides utility tool endpoints including: -- Web scraping and content extraction - DNS lookups """ from fastapi import APIRouter, HTTPException, status from src.controllers.base import BaseController from src.logging_config import get_logger -from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse -from src.web_scraper.service import WebScraperService -from src.web_scraper.exceptions import FetchError, ScrapingError from src.dns.schemas import DNSLookupRequest, DNSLookupResponse from src.dns.service import DNSService from src.dns.exceptions import DNSQueryError @@ -24,79 +20,17 @@ class ToolsController(BaseController): Controller for utility tools Provides endpoints for: - - Web scraping and content extraction - DNS lookups """ def __init__(self): super().__init__(prefix="/tools", tags=["Tools"]) - # Initialize services (could be dependency injected for testing) - self.scraper_service = WebScraperService() self.dns_service = DNSService() def create_router(self) -> APIRouter: """Create and configure the router""" router = APIRouter(prefix=self.prefix, tags=self.tags) - @router.post( - "/scrape", - response_model=WebScraperResponse, - status_code=status.HTTP_200_OK, - summary="Scrape website content", - description=""" - Scrape and extract main content from a website. - - Uses trafilatura for intelligent content extraction (articles, blog posts, documentation), - with BeautifulSoup as fallback. Perfect for feeding webpage content to LLMs. - - **Features:** - - Intelligent main content extraction - - Removes navigation, ads, footers - - Optional link extraction - - Configurable content length limits - - **Rate Limiting:** None (internal network use only) - """ - ) - async def scrape_website(request: WebScraperRequest) -> WebScraperResponse: - """ - Scrape a website and extract its main content - - Args: - request: Scraping request with URL and options - - Returns: - Extracted content with metadata - - Raises: - HTTPException: 400 for fetch errors, 500 for processing errors - """ - try: - logger.info(f"Received scrape request for: {request.url}") - result = await self.scraper_service.scrape_url(request) - return result - - except FetchError as e: - logger.warning(f"Fetch failed: {str(e)}") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to fetch URL: {str(e)}" - ) - - except ScrapingError as e: - logger.error(f"Scraping failed: {str(e)}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to extract content: {str(e)}" - ) - - except Exception as e: - logger.error(f"Unexpected error: {str(e)}", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="An unexpected error occurred" - ) - @router.post( "/dns/lookup", response_model=DNSLookupResponse, diff --git a/src/web_scraper/__init__.py b/src/web_scraper/__init__.py deleted file mode 100644 index fa69ae4..0000000 --- a/src/web_scraper/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -Web scraper module for extracting content from websites -""" -from src.web_scraper.router import router -from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse -from src.web_scraper.service import WebScraperService - -__all__ = [ - "router", - "WebScraperRequest", - "WebScraperResponse", - "WebScraperService", -] diff --git a/src/web_scraper/config.py b/src/web_scraper/config.py deleted file mode 100644 index 70c896c..0000000 --- a/src/web_scraper/config.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -Configuration for web scraper module -""" -from pydantic_settings import BaseSettings -from functools import lru_cache - - -class WebScraperSettings(BaseSettings): - """Web scraper specific settings""" - - # HTTP client configuration - request_timeout: int = 30 - max_redirects: int = 5 - user_agent: str = "Mozilla/5.0 (compatible; CoreCode/1.0)" - - # Content extraction - default_max_length: int = 10000 - max_links_to_extract: int = 50 - - # Rate limiting (future use) - rate_limit_enabled: bool = False - requests_per_minute: int = 60 - - class Config: - env_prefix = "WEB_SCRAPER_" - case_sensitive = False - - -@lru_cache() -def get_web_scraper_settings() -> WebScraperSettings: - """Cached web scraper settings instance""" - return WebScraperSettings() diff --git a/src/web_scraper/exceptions.py b/src/web_scraper/exceptions.py deleted file mode 100644 index 3fb5443..0000000 --- a/src/web_scraper/exceptions.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Custom exceptions for web scraper module -""" - - -class WebScraperException(Exception): - """Base exception for web scraper module""" - pass - - -class FetchError(WebScraperException): - """Raised when URL fetch fails""" - pass - - -class ScrapingError(WebScraperException): - """Raised when content extraction fails""" - pass diff --git a/src/web_scraper/router.py b/src/web_scraper/router.py deleted file mode 100644 index afb3065..0000000 --- a/src/web_scraper/router.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -API routes for web scraper module -""" -from fastapi import APIRouter, HTTPException, status - -from src.logging_config import get_logger -from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse -from src.web_scraper.service import WebScraperService -from src.web_scraper.exceptions import FetchError, ScrapingError - -logger = get_logger(__name__) - -router = APIRouter( - prefix="/web-scraper", - tags=["Web Scraper"] -) - -# Initialize service (could be dependency injected for testing) -scraper_service = WebScraperService() - - -@router.post( - "/scrape", - response_model=WebScraperResponse, - status_code=status.HTTP_200_OK, - summary="Scrape website content", - description=""" - Scrape and extract main content from a website. - - Uses trafilatura for intelligent content extraction (articles, blog posts, documentation), - with BeautifulSoup as fallback. Perfect for feeding webpage content to LLMs. - - **Features:** - - Intelligent main content extraction - - Removes navigation, ads, footers - - Optional link extraction - - Configurable content length limits - - **Rate Limiting:** None (internal network use only) - """ -) -async def scrape_website(request: WebScraperRequest) -> WebScraperResponse: - """ - Scrape a website and extract its main content - - Args: - request: Scraping request with URL and options - - Returns: - Extracted content with metadata - - Raises: - HTTPException: 400 for fetch errors, 500 for processing errors - """ - try: - logger.info(f"Received scrape request for: {request.url}") - result = await scraper_service.scrape_url(request) - return result - - except FetchError as e: - logger.warning(f"Fetch failed: {str(e)}") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to fetch URL: {str(e)}" - ) - - except ScrapingError as e: - logger.error(f"Scraping failed: {str(e)}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to extract content: {str(e)}" - ) - - except Exception as e: - logger.error(f"Unexpected error: {str(e)}", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="An unexpected error occurred" - ) diff --git a/src/web_scraper/schemas.py b/src/web_scraper/schemas.py deleted file mode 100644 index 9afde82..0000000 --- a/src/web_scraper/schemas.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Pydantic schemas for web scraper module -""" -from pydantic import HttpUrl, Field -from typing import Optional -from datetime import datetime -from src.base_schema import BaseSchema - - -class WebScraperRequest(BaseSchema): - """Request model for web scraping""" - - url: HttpUrl = Field( - ..., - description="The URL to scrape", - examples=["https://example.com/article"] - ) - - extract_main_content: bool = Field( - default=True, - description="Use intelligent content extraction (trafilatura) vs raw HTML parsing" - ) - - include_links: bool = Field( - default=False, - description="Include list of links found on the page" - ) - - max_length: Optional[int] = Field( - default=10000, - ge=100, - le=100000, - description="Maximum content length to return (100-100000 chars)" - ) - - -class WebScraperResponse(BaseSchema): - """Response model for web scraping""" - - url: str = Field( - ..., - description="The scraped URL" - ) - - title: Optional[str] = Field( - default=None, - description="Page title extracted from tag" - ) - - content: str = Field( - ..., - description="Extracted page content" - ) - - extracted_at: datetime = Field( - ..., - description="UTC timestamp when content was extracted" - ) - - content_length: int = Field( - ..., - ge=0, - description="Length of extracted content in characters" - ) - - links: Optional[list[str]] = Field( - default=None, - description="List of HTTP(S) links found on the page (max 50)" - ) diff --git a/src/web_scraper/service.py b/src/web_scraper/service.py deleted file mode 100644 index f45468a..0000000 --- a/src/web_scraper/service.py +++ /dev/null @@ -1,213 +0,0 @@ -""" -Business logic for web scraper module -""" -import httpx -from bs4 import BeautifulSoup -import trafilatura -from datetime import datetime, timezone -from typing import Optional - -from src.logging_config import get_logger -from src.web_scraper.config import get_web_scraper_settings -from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse -from src.web_scraper.exceptions import ScrapingError, FetchError - -logger = get_logger(__name__) - - -class WebScraperService: - """Service class for web scraping operations""" - - def __init__(self): - self.settings = get_web_scraper_settings() - - async def scrape_url(self, request: WebScraperRequest) -> WebScraperResponse: - """ - Scrape and extract content from a URL - - Args: - request: Scraping request parameters - - Returns: - Extracted content with metadata - - Raises: - FetchError: If URL cannot be fetched - ScrapingError: If content extraction fails - """ - url_str = str(request.url) - logger.info(f"Starting scrape for URL: {url_str}") - - try: - # Fetch the webpage - html_content = await self._fetch_url(url_str) - - # Extract content based on settings - if request.extract_main_content: - content = self._extract_main_content(html_content, request.include_links) - else: - content = self._extract_basic_content(html_content) - - # Extract metadata - title = self._extract_title(html_content) - links = self._extract_links(html_content) if request.include_links else None - - # Clean and truncate content - content = self._clean_content(content) - if request.max_length and len(content) > request.max_length: - content = content[:request.max_length] + "\n\n[Content truncated...]" - logger.debug(f"Content truncated to {request.max_length} characters") - - logger.info(f"Successfully scraped {len(content)} characters from {url_str}") - - return WebScraperResponse( - url=url_str, - title=title, - content=content, - extracted_at=datetime.now(timezone.utc), - content_length=len(content), - links=links - ) - - except FetchError: - raise - except Exception as e: - logger.error(f"Scraping failed for {url_str}: {str(e)}", exc_info=True) - raise ScrapingError(f"Failed to scrape content: {str(e)}") - - async def _fetch_url(self, url: str) -> str: - """ - Fetch HTML content from URL - - Args: - url: URL to fetch - - Returns: - HTML content as string - - Raises: - FetchError: If fetch fails - """ - try: - async with httpx.AsyncClient( - timeout=self.settings.request_timeout, - follow_redirects=True, - max_redirects=self.settings.max_redirects - ) as client: - logger.debug(f"Fetching URL: {url}") - response = await client.get( - url, - headers={"User-Agent": self.settings.user_agent} - ) - response.raise_for_status() - logger.debug(f"Fetched {len(response.text)} bytes from {url}") - return response.text - - except httpx.HTTPStatusError as e: - logger.error(f"HTTP error {e.response.status_code} for {url}") - raise FetchError(f"HTTP {e.response.status_code}: {e.response.reason_phrase}") - except httpx.RequestError as e: - logger.error(f"Request error for {url}: {str(e)}") - raise FetchError(f"Failed to fetch URL: {str(e)}") - - def _extract_main_content(self, html: str, include_links: bool = False) -> str: - """ - Extract main content using trafilatura (intelligent extraction) - - Args: - html: Raw HTML content - include_links: Whether to preserve links in output - - Returns: - Extracted content - """ - logger.debug("Extracting main content with trafilatura") - content = trafilatura.extract( - html, - include_links=include_links, - include_images=False, - output_format='txt', - no_fallback=False - ) - - # Fallback to BeautifulSoup if trafilatura fails - if not content: - logger.debug("Trafilatura extraction failed, falling back to BeautifulSoup") - content = self._extract_basic_content(html) - - return content - - def _extract_basic_content(self, html: str) -> str: - """ - Extract content using basic BeautifulSoup parsing - - Args: - html: Raw HTML content - - Returns: - Extracted text content - """ - logger.debug("Extracting content with BeautifulSoup") - soup = BeautifulSoup(html, 'html.parser') - - # Remove unwanted elements - for element in soup(["script", "style", "nav", "footer", "header", "aside"]): - element.decompose() - - # Extract text - text = soup.get_text(separator='\n', strip=True) - return text - - def _extract_title(self, html: str) -> Optional[str]: - """ - Extract page title from HTML - - Args: - html: Raw HTML content - - Returns: - Page title or None - """ - soup = BeautifulSoup(html, 'html.parser') - title = soup.title.string if soup.title else None - if title: - title = title.strip() - logger.debug(f"Extracted title: {title}") - return title - - def _extract_links(self, html: str) -> list[str]: - """ - Extract HTTP(S) links from HTML - - Args: - html: Raw HTML content - - Returns: - List of absolute HTTP(S) URLs - """ - soup = BeautifulSoup(html, 'html.parser') - links = [ - a.get('href') - for a in soup.find_all('a', href=True) - if a.get('href', '').startswith('http') - ] - - # Limit number of links - links = links[:self.settings.max_links_to_extract] - logger.debug(f"Extracted {len(links)} links") - return links - - def _clean_content(self, content: str) -> str: - """ - Clean and normalize extracted content - - Args: - content: Raw extracted content - - Returns: - Cleaned content - """ - # Remove empty lines and normalize whitespace - lines = [line.strip() for line in content.split('\n') if line.strip()] - cleaned = '\n'.join(lines) - return cleaned