70 lines
1.6 KiB
Python
70 lines
1.6 KiB
Python
"""
|
|
Pydantic schemas for web scraper module
|
|
"""
|
|
from pydantic import HttpUrl, Field
|
|
from typing import Optional
|
|
from datetime import datetime
|
|
from src.base_schema import BaseSchema
|
|
|
|
|
|
class WebScraperRequest(BaseSchema):
|
|
"""Request model for web scraping"""
|
|
|
|
url: HttpUrl = Field(
|
|
...,
|
|
description="The URL to scrape",
|
|
examples=["https://example.com/article"]
|
|
)
|
|
|
|
extract_main_content: bool = Field(
|
|
default=True,
|
|
description="Use intelligent content extraction (trafilatura) vs raw HTML parsing"
|
|
)
|
|
|
|
include_links: bool = Field(
|
|
default=False,
|
|
description="Include list of links found on the page"
|
|
)
|
|
|
|
max_length: Optional[int] = Field(
|
|
default=10000,
|
|
ge=100,
|
|
le=100000,
|
|
description="Maximum content length to return (100-100000 chars)"
|
|
)
|
|
|
|
|
|
class WebScraperResponse(BaseSchema):
|
|
"""Response model for web scraping"""
|
|
|
|
url: str = Field(
|
|
...,
|
|
description="The scraped URL"
|
|
)
|
|
|
|
title: Optional[str] = Field(
|
|
default=None,
|
|
description="Page title extracted from <title> tag"
|
|
)
|
|
|
|
content: str = Field(
|
|
...,
|
|
description="Extracted page content"
|
|
)
|
|
|
|
extracted_at: datetime = Field(
|
|
...,
|
|
description="UTC timestamp when content was extracted"
|
|
)
|
|
|
|
content_length: int = Field(
|
|
...,
|
|
ge=0,
|
|
description="Length of extracted content in characters"
|
|
)
|
|
|
|
links: Optional[list[str]] = Field(
|
|
default=None,
|
|
description="List of HTTP(S) links found on the page (max 50)"
|
|
)
|