Initial commit: core-api service extraction from portainer-core

src/web_scraper/router.py

@@ -0,0 +1,79 @@
+"""
+API routes for web scraper module
+"""
+from fastapi import APIRouter, HTTPException, status
+
+from src.logging_config import get_logger
+from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse
+from src.web_scraper.service import WebScraperService
+from src.web_scraper.exceptions import FetchError, ScrapingError
+
+logger = get_logger(__name__)
+
+router = APIRouter(
+    prefix="/web-scraper",
+    tags=["Web Scraper"]
+)
+
+# Initialize service (could be dependency injected for testing)
+scraper_service = WebScraperService()
+
+
+@router.post(
+    "/scrape",
+    response_model=WebScraperResponse,
+    status_code=status.HTTP_200_OK,
+    summary="Scrape website content",
+    description="""
+    Scrape and extract main content from a website.
+
+    Uses trafilatura for intelligent content extraction (articles, blog posts, documentation),
+    with BeautifulSoup as fallback. Perfect for feeding webpage content to LLMs.
+
+    **Features:**
+    - Intelligent main content extraction
+    - Removes navigation, ads, footers
+    - Optional link extraction
+    - Configurable content length limits
+
+    **Rate Limiting:** None (internal network use only)
+    """
+)
+async def scrape_website(request: WebScraperRequest) -> WebScraperResponse:
+    """
+    Scrape a website and extract its main content
+
+    Args:
+        request: Scraping request with URL and options
+
+    Returns:
+        Extracted content with metadata
+
+    Raises:
+        HTTPException: 400 for fetch errors, 500 for processing errors
+    """
+    try:
+        logger.info(f"Received scrape request for: {request.url}")
+        result = await scraper_service.scrape_url(request)
+        return result
+
+    except FetchError as e:
+        logger.warning(f"Fetch failed: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Failed to fetch URL: {str(e)}"
+        )
+
+    except ScrapingError as e:
+        logger.error(f"Scraping failed: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to extract content: {str(e)}"
+        )
+
+    except Exception as e:
+        logger.error(f"Unexpected error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="An unexpected error occurred"
+        )