80 lines
2.3 KiB
Python
80 lines
2.3 KiB
Python
"""
|
|
API routes for web scraper module
|
|
"""
|
|
from fastapi import APIRouter, HTTPException, status
|
|
|
|
from src.logging_config import get_logger
|
|
from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse
|
|
from src.web_scraper.service import WebScraperService
|
|
from src.web_scraper.exceptions import FetchError, ScrapingError
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
router = APIRouter(
|
|
prefix="/web-scraper",
|
|
tags=["Web Scraper"]
|
|
)
|
|
|
|
# Initialize service (could be dependency injected for testing)
|
|
scraper_service = WebScraperService()
|
|
|
|
|
|
@router.post(
|
|
"/scrape",
|
|
response_model=WebScraperResponse,
|
|
status_code=status.HTTP_200_OK,
|
|
summary="Scrape website content",
|
|
description="""
|
|
Scrape and extract main content from a website.
|
|
|
|
Uses trafilatura for intelligent content extraction (articles, blog posts, documentation),
|
|
with BeautifulSoup as fallback. Perfect for feeding webpage content to LLMs.
|
|
|
|
**Features:**
|
|
- Intelligent main content extraction
|
|
- Removes navigation, ads, footers
|
|
- Optional link extraction
|
|
- Configurable content length limits
|
|
|
|
**Rate Limiting:** None (internal network use only)
|
|
"""
|
|
)
|
|
async def scrape_website(request: WebScraperRequest) -> WebScraperResponse:
|
|
"""
|
|
Scrape a website and extract its main content
|
|
|
|
Args:
|
|
request: Scraping request with URL and options
|
|
|
|
Returns:
|
|
Extracted content with metadata
|
|
|
|
Raises:
|
|
HTTPException: 400 for fetch errors, 500 for processing errors
|
|
"""
|
|
try:
|
|
logger.info(f"Received scrape request for: {request.url}")
|
|
result = await scraper_service.scrape_url(request)
|
|
return result
|
|
|
|
except FetchError as e:
|
|
logger.warning(f"Fetch failed: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=f"Failed to fetch URL: {str(e)}"
|
|
)
|
|
|
|
except ScrapingError as e:
|
|
logger.error(f"Scraping failed: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Failed to extract content: {str(e)}"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Unexpected error: {str(e)}", exc_info=True)
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail="An unexpected error occurred"
|
|
)
|