Remove obsolete web scraper module

Web scraping functionality is no longer needed. Removes:
- src/web_scraper/ directory (6 files)
- /web-scraper/scrape endpoint from tools controller
- References from health controller endpoints list

Tools controller now only contains DNS lookup functionality.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/controllers/tools_controller.py

@@ -2,16 +2,12 @@
 Tools Controller
 
 Provides utility tool endpoints including:
-- Web scraping and content extraction
 - DNS lookups
 """
 from fastapi import APIRouter, HTTPException, status
 
 from src.controllers.base import BaseController
 from src.logging_config import get_logger
-from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse
-from src.web_scraper.service import WebScraperService
-from src.web_scraper.exceptions import FetchError, ScrapingError
 from src.dns.schemas import DNSLookupRequest, DNSLookupResponse
 from src.dns.service import DNSService
 from src.dns.exceptions import DNSQueryError
@@ -24,79 +20,17 @@ class ToolsController(BaseController):
     Controller for utility tools
 
     Provides endpoints for:
-    - Web scraping and content extraction
     - DNS lookups
     """
 
     def __init__(self):
         super().__init__(prefix="/tools", tags=["Tools"])
-        # Initialize services (could be dependency injected for testing)
-        self.scraper_service = WebScraperService()
         self.dns_service = DNSService()
 
     def create_router(self) -> APIRouter:
         """Create and configure the router"""
         router = APIRouter(prefix=self.prefix, tags=self.tags)
 
-        @router.post(
-            "/scrape",
-            response_model=WebScraperResponse,
-            status_code=status.HTTP_200_OK,
-            summary="Scrape website content",
-            description="""
-            Scrape and extract main content from a website.
-
-            Uses trafilatura for intelligent content extraction (articles, blog posts, documentation),
-            with BeautifulSoup as fallback. Perfect for feeding webpage content to LLMs.
-
-            **Features:**
-            - Intelligent main content extraction
-            - Removes navigation, ads, footers
-            - Optional link extraction
-            - Configurable content length limits
-
-            **Rate Limiting:** None (internal network use only)
-            """
-        )
-        async def scrape_website(request: WebScraperRequest) -> WebScraperResponse:
-            """
-            Scrape a website and extract its main content
-
-            Args:
-                request: Scraping request with URL and options
-
-            Returns:
-                Extracted content with metadata
-
-            Raises:
-                HTTPException: 400 for fetch errors, 500 for processing errors
-            """
-            try:
-                logger.info(f"Received scrape request for: {request.url}")
-                result = await self.scraper_service.scrape_url(request)
-                return result
-
-            except FetchError as e:
-                logger.warning(f"Fetch failed: {str(e)}")
-                raise HTTPException(
-                    status_code=status.HTTP_400_BAD_REQUEST,
-                    detail=f"Failed to fetch URL: {str(e)}"
-                )
-
-            except ScrapingError as e:
-                logger.error(f"Scraping failed: {str(e)}")
-                raise HTTPException(
-                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                    detail=f"Failed to extract content: {str(e)}"
-                )
-
-            except Exception as e:
-                logger.error(f"Unexpected error: {str(e)}", exc_info=True)
-                raise HTTPException(
-                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                    detail="An unexpected error occurred"
-                )
-
         @router.post(
             "/dns/lookup",
             response_model=DNSLookupResponse,