fix: stream content fetches so the 5MB cap aborts the download

ContentExtractor._fetch used client.get(), buffering the whole body in
memory before the MAX_RESPONSE_BYTES check truncated it - the cap
protected Trafilatura but not memory/bandwidth (a multi-hundred-MB URL
was still fully downloaded, on up to max_urls_per_batch concurrent
fetches, bounded only by the read timeout).

Fetches now stream via client.stream + aiter_bytes and close the
connection as soon as the cap is reached; charset still comes from the
Content-Type header, available before the body is read.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L

src/clients/content_extractor.py

@@ -38,8 +38,9 @@ DEFAULT_HEADERS = {
     "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) LibraryDesk-ContentExtractor"
 }
 
-# Responses larger than this are truncated before parsing (Trafilatura on a
-# multi-hundred-MB response would pin a worker thread and exhaust memory).
+# Downloads are streamed and ABORTED past this many bytes (protects memory
+# and bandwidth during the fetch, and keeps Trafilatura from pinning a
+# worker thread on a multi-hundred-MB response).
 MAX_RESPONSE_BYTES = 5 * 1024 * 1024
 
 
@@ -103,25 +104,49 @@ class ContentExtractor:
         """
         Fetch a URL asynchronously under real connect/read timeouts.
 
+        The body is STREAMED and the download is aborted as soon as
+        MAX_RESPONSE_BYTES have been received, so the cap bounds memory and
+        bandwidth during the download itself (the old implementation
+        buffered the entire response before truncating, so a
+        multi-hundred-MB URL was still fully downloaded).
+
         Returns:
-            Response text (truncated to MAX_RESPONSE_BYTES) or None when the
+            Response text (capped at MAX_RESPONSE_BYTES) or None when the
             response is empty / not OK.
 
         Raises:
             httpx.HTTPError subclasses on timeout/network errors.
         """
-        response = await self._http.get(url)
-        if response.status_code != 200 or not response.content:
-            logger.debug(f"Fetch returned status {response.status_code} for {url}")
-            return None
-        if len(response.content) > MAX_RESPONSE_BYTES:
-            logger.warning(
-                f"Response for {url} exceeds {MAX_RESPONSE_BYTES} bytes; truncating"
-            )
-            return response.content[:MAX_RESPONSE_BYTES].decode(
-                response.encoding or "utf-8", errors="replace"
-            )
-        return response.text
+        async with self._http.stream("GET", url) as response:
+            if response.status_code != 200:
+                logger.debug(
+                    f"Fetch returned status {response.status_code} for {url}"
+                )
+                return None
+
+            chunks: List[bytes] = []
+            received = 0
+            truncated = False
+            async for chunk in response.aiter_bytes():
+                if received + len(chunk) >= MAX_RESPONSE_BYTES:
+                    chunks.append(chunk[: MAX_RESPONSE_BYTES - received])
+                    truncated = True
+                    break
+                chunks.append(chunk)
+                received += len(chunk)
+
+            body = b"".join(chunks)
+            if not body:
+                return None
+            if truncated:
+                logger.warning(
+                    f"Response for {url} exceeds {MAX_RESPONSE_BYTES} bytes; "
+                    "download aborted and body truncated"
+                )
+            # charset comes from the Content-Type header, available before
+            # the body is read.
+            encoding = response.charset_encoding or "utf-8"
+            return body.decode(encoding, errors="replace")
 
     def _log_queue_depth(self, context: str) -> None:
         """Log thread-pool queue depth so extraction backpressure is visible."""