feat(provider): support multiple ChatGPT subscriptions with usage

This commit is contained in:
Alexandre Teixeira
2026-09-22 13:12:19 +01:00
parent 45330097b8
commit ed7ccfd584
31 changed files with 3281 additions and 215 deletions
+410 -10
View File
@@ -9,10 +9,13 @@ from __future__ import annotations
import base64
import json
import math
import os
import re
import threading
import time
from typing import Any, Dict, Optional
import unicodedata
from typing import Any, Dict, List, Optional
import httpx
from fastapi import HTTPException
@@ -22,6 +25,16 @@ DEFAULT_CHATGPT_SUBSCRIPTION_BASE_URL = (
or "https://chatgpt.com/backend-api/codex"
)
CHATGPT_SUBSCRIPTION_PROVIDER = "chatgpt-subscription"
# Legacy single-account endpoint/auth name. Rows provisioned before
# multi-account support keep this name and stay functional.
CHATGPT_SUBSCRIPTION_LEGACY_NAME = "ChatGPT Subscription"
CHATGPT_ACCOUNT_LABEL_MAX_LENGTH = 40
# Read-only account usage (rate-limit windows) on the authenticated ChatGPT
# backend. Mirrors openai/codex ``backend-client`` ``PathStyle::ChatGptApi``:
# ``{base}/wham/usage`` where base is ``https://chatgpt.com/backend-api``.
CHATGPT_USAGE_URL = "https://chatgpt.com/backend-api/wham/usage"
CHATGPT_USAGE_TIMEOUT_SECONDS = 8.0
CHATGPT_USAGE_CACHE_TTL_SECONDS = 45.0
CHATGPT_OAUTH_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann"
CHATGPT_OAUTH_TOKEN_URL = "https://auth.openai.com/oauth/token"
CHATGPT_OAUTH_ISSUER = "https://auth.openai.com"
@@ -87,6 +100,53 @@ def chatgpt_headers(access_token: Optional[str]) -> Dict[str, str]:
return headers
# ── Account labels ──────────────────────────────────────────────────────────
_LABEL_WHITESPACE_RE = re.compile(r"\s+")
def normalize_account_label(raw: Any) -> str:
"""Return a trimmed, display-safe account label ("" when absent).
Labels are purely cosmetic: they never participate in authentication or
authorization (stable auth/endpoint ids do). Control characters are
stripped, whitespace collapsed and the length bounded so the label is safe
to echo in Settings, the model picker and provenance metadata.
"""
if raw is None:
return ""
text = str(raw)
text = "".join(ch for ch in text if unicodedata.category(ch)[0] != "C")
text = _LABEL_WHITESPACE_RE.sub(" ", text).strip()
if len(text) > CHATGPT_ACCOUNT_LABEL_MAX_LENGTH:
raise ValueError(
f"Account label must be at most {CHATGPT_ACCOUNT_LABEL_MAX_LENGTH} characters."
)
return text
def endpoint_name_for_label(label: str) -> str:
"""User-visible endpoint name for a ChatGPT account label."""
label = (label or "").strip()
if not label:
return CHATGPT_SUBSCRIPTION_LEGACY_NAME
return f"ChatGPT · {label}"
def account_label_from_name(name: Optional[str], fallback: str = "") -> str:
"""Inverse of :func:`endpoint_name_for_label` for display purposes."""
text = (name or "").strip()
if text.startswith("ChatGPT · "):
return text[len("ChatGPT · "):].strip() or fallback
if text == CHATGPT_SUBSCRIPTION_LEGACY_NAME:
return fallback
return text or fallback
def labels_conflict(a: str, b: str) -> bool:
return bool(a) and bool(b) and a.casefold() == b.casefold()
def fetch_available_models(access_token: str, timeout: float = 10.0) -> list[str]:
if not access_token:
return []
@@ -135,14 +195,8 @@ def _raise_for_oauth_response(response: httpx.Response, action: str) -> None:
err = payload.get("error") if isinstance(payload, dict) else None
if isinstance(err, dict):
code = str(err.get("code") or err.get("type") or "").strip()
msg = err.get("message")
if msg:
message = f"ChatGPT Subscription {action} failed: {msg}"
elif isinstance(err, str):
code = err.strip()
desc = payload.get("error_description") or payload.get("message")
if desc:
message = f"ChatGPT Subscription {action} failed: {desc}"
except Exception:
pass
if response.status_code == 429:
@@ -259,8 +313,7 @@ def resolve_runtime_credentials(auth_id: str, owner: Optional[str] = None, *, fo
ProviderAuthSession.id == auth_id,
ProviderAuthSession.provider == CHATGPT_SUBSCRIPTION_PROVIDER,
)
if owner:
q = q.filter(ProviderAuthSession.owner == owner)
q = q.filter(ProviderAuthSession.owner == owner)
row = q.first()
if row is None:
raise ChatGPTSubscriptionAuthNotFound("ChatGPT Subscription credentials were not found for this user.")
@@ -291,12 +344,359 @@ def resolve_runtime_credentials(auth_id: str, owner: Optional[str] = None, *, fo
db.close()
def find_owned_auth_session(db, auth_id: str, owner: Optional[str]):
"""Return the owner-scoped ChatGPT ProviderAuthSession row or None.
OAuth credentials belong to exactly one owner, including the legacy
anonymous owner. Labels are never used for lookup.
"""
ProviderAuthSession, _SessionLocal, _now = _database_handles()
auth_id = (auth_id or "").strip()
if not auth_id:
return None
q = db.query(ProviderAuthSession).filter(
ProviderAuthSession.id == auth_id,
ProviderAuthSession.provider == CHATGPT_SUBSCRIPTION_PROVIDER,
)
return q.filter(ProviderAuthSession.owner == owner).first()
def chatgpt_account_id_from_token(access_token: str) -> Optional[str]:
"""Extract the ChatGPT account id claim from an access token, if any."""
try:
payload = _decode_jwt_payload(access_token)
except Exception:
return None
auth_claims = payload.get("https://api.openai.com/auth")
if isinstance(auth_claims, dict):
account_id = auth_claims.get("chatgpt_account_id")
if isinstance(account_id, str) and account_id.strip():
return account_id.strip()
return None
def usage_request_headers(access_token: str) -> Dict[str, str]:
headers = {
"Accept": "application/json",
"User-Agent": "Odysseus ChatGPT Subscription",
"Authorization": f"Bearer {access_token}",
}
account_id = chatgpt_account_id_from_token(access_token)
if account_id:
headers["ChatGPT-Account-Id"] = account_id
return headers
class ChatGPTUsageUnavailable(ChatGPTSubscriptionError):
"""Usage telemetry could not be read; the model endpoint is unaffected."""
def __init__(self, reason: str, message: str, *, status_code: Optional[int] = None):
super().__init__(message)
self.reason = reason
self.status_code = status_code
def _coerce_number(value: Any) -> Optional[float]:
if isinstance(value, bool):
return None
if isinstance(value, (int, float, str)):
try:
number = float(value)
return number if math.isfinite(number) else None
except (ValueError, OverflowError):
return None
return None
def _coerce_int(value: Any) -> Optional[int]:
number = _coerce_number(value)
if number is None:
return None
try:
return int(number)
except (OverflowError, ValueError):
return None
def _optional_str(value: Any) -> Optional[str]:
if isinstance(value, str):
text = value.strip()
return text or None
return None
def window_minutes_from_seconds(seconds: Any) -> Optional[int]:
"""Codex-compatible ceil(seconds / 60); None for missing/non-positive."""
value = _coerce_int(seconds)
if value is None or value <= 0:
return None
return (value + 59) // 60
def friendly_window_name(window_minutes: Optional[int]) -> str:
"""Compact label derived from the actual window length (never assumed)."""
if not window_minutes or window_minutes <= 0:
return "LIMIT"
if window_minutes % (7 * 24 * 60) == 0:
weeks = window_minutes // (7 * 24 * 60)
return "WEEK" if weeks == 1 else f"{weeks}W"
if window_minutes % (24 * 60) == 0:
return f"{window_minutes // (24 * 60)}D"
if window_minutes % 60 == 0:
return f"{window_minutes // 60}H"
return f"{window_minutes}M"
def normalize_usage_window(raw: Any, kind: str) -> Optional[Dict[str, Any]]:
"""Normalize one ``primary_window``/``secondary_window`` snapshot.
Upstream fields (openai/codex ``RateLimitWindowSnapshot``): ``used_percent``,
``limit_window_seconds``, ``reset_after_seconds``, ``reset_at``. Absent
values stay ``None``; nothing is invented.
"""
if not isinstance(raw, dict):
return None
used_percent = _coerce_number(raw.get("used_percent"))
if used_percent is not None:
used_percent = max(0.0, min(100.0, used_percent))
window_minutes = window_minutes_from_seconds(raw.get("limit_window_seconds"))
if window_minutes is None:
window_minutes = _coerce_int(raw.get("window_minutes"))
if window_minutes is not None and window_minutes <= 0:
window_minutes = None
resets_at = _coerce_int(raw.get("reset_at"))
if resets_at is None:
resets_at = _coerce_int(raw.get("resets_at"))
if resets_at is not None and resets_at <= 0:
resets_at = None
reset_after_seconds = _coerce_int(raw.get("reset_after_seconds"))
if reset_after_seconds is not None and reset_after_seconds < 0:
reset_after_seconds = None
return {
"kind": kind,
"name": friendly_window_name(window_minutes),
"used_percent": used_percent,
"remaining_percent": (None if used_percent is None else round(100.0 - used_percent, 2)),
"window_minutes": window_minutes,
"resets_at": resets_at,
"reset_after_seconds": reset_after_seconds,
}
def _normalize_rate_limit_details(raw: Any) -> Dict[str, Any]:
details = raw if isinstance(raw, dict) else {}
windows: List[Dict[str, Any]] = []
# Preserve new window kinds without assigning a duration to their names.
keys = ["primary_window", "secondary_window"]
keys.extend(key for key in details if key.endswith("_window") and key not in keys)
for key in keys:
window = normalize_usage_window(details.get(key), key[:-7])
if window is not None:
windows.append(window)
allowed = details.get("allowed")
limit_reached = details.get("limit_reached")
return {
"allowed": allowed if isinstance(allowed, bool) else None,
"limit_reached": limit_reached if isinstance(limit_reached, bool) else None,
"windows": windows,
}
def normalize_usage_payload(payload: Any) -> Dict[str, Any]:
"""Normalize a ``GET /wham/usage`` JSON body into Odysseus' safe contract.
Returns only non-credential fields. The main Codex limit is reported as
``limit_id == "codex"`` (as openai/codex does); each entry of
``additional_rate_limits`` becomes its own bucket keyed by
``metered_feature``. Unknown fields are ignored, unknown buckets kept.
"""
if not isinstance(payload, dict):
raise ChatGPTUsageUnavailable("malformed", "ChatGPT usage response was not a JSON object.")
limits: List[Dict[str, Any]] = []
main = _normalize_rate_limit_details(payload.get("rate_limit"))
limits.append({
"limit_id": "codex",
"limit_name": None,
"normal_model_slug": None,
"allowed": main["allowed"],
"limit_reached": main["limit_reached"],
"windows": main["windows"],
})
additional = payload.get("additional_rate_limits")
if isinstance(additional, list):
for entry in additional:
if not isinstance(entry, dict):
continue
details = _normalize_rate_limit_details(entry.get("rate_limit"))
limit_name = _optional_str(entry.get("limit_name"))
limit_id = _optional_str(entry.get("metered_feature")) or limit_name
if not limit_id and not details["windows"]:
continue
limits.append({
"limit_id": limit_id or "additional",
"limit_name": limit_name,
"normal_model_slug": _optional_str(entry.get("normal_model_slug")),
"allowed": details["allowed"],
"limit_reached": details["limit_reached"],
"windows": details["windows"],
})
reached = payload.get("rate_limit_reached_type")
if isinstance(reached, dict):
reached = _optional_str(reached.get("type") or reached.get("kind"))
else:
reached = _optional_str(reached)
ordinary_usage_allowed = main["allowed"]
return {
"account_id": _optional_str(payload.get("account_id")),
"plan_type": _optional_str(payload.get("plan_type")),
"ordinary_usage_allowed": ordinary_usage_allowed,
"rate_limit_reached_type": reached,
"limits": limits,
}
def fetch_usage_payload(access_token: str, timeout: float = CHATGPT_USAGE_TIMEOUT_SECONDS) -> Dict[str, Any]:
"""Read the raw usage JSON for one access token; classify failures."""
if not access_token:
raise ChatGPTUsageUnavailable("reauth", "ChatGPT Subscription has no access token.")
try:
response = httpx.get(CHATGPT_USAGE_URL, headers=usage_request_headers(access_token), timeout=timeout)
except httpx.TimeoutException as exc:
raise ChatGPTUsageUnavailable("timeout", "ChatGPT usage request timed out.") from exc
except httpx.HTTPError as exc:
raise ChatGPTUsageUnavailable("network", "ChatGPT usage request failed.") from exc
status = response.status_code
if status in (401, 403):
raise ChatGPTUsageUnavailable(
"reauth",
"ChatGPT rejected the usage request; the account may need reconnecting.",
status_code=status,
)
if status == 429:
raise ChatGPTUsageUnavailable("rate_limited", "ChatGPT usage is temporarily rate limited.", status_code=status)
if status >= 500:
raise ChatGPTUsageUnavailable("upstream", f"ChatGPT usage service returned HTTP {status}.", status_code=status)
if status != 200:
raise ChatGPTUsageUnavailable("upstream", f"ChatGPT usage request returned HTTP {status}.", status_code=status)
try:
data = response.json()
except Exception as exc:
raise ChatGPTUsageUnavailable("malformed", "ChatGPT usage response was not valid JSON.") from exc
if not isinstance(data, dict):
raise ChatGPTUsageUnavailable("malformed", "ChatGPT usage response was not a JSON object.")
return data
class UsageCache:
"""Short per-auth-session cache for normalized usage snapshots."""
def __init__(self, ttl_seconds: float = CHATGPT_USAGE_CACHE_TTL_SECONDS, time_func=time.monotonic, max_entries: int = 256):
self._ttl = float(ttl_seconds)
self._max_entries = max(1, int(max_entries))
self._time = time_func
self._entries: Dict[str, tuple] = {}
self._lock = threading.Lock()
def get(self, auth_id: str) -> Optional[Dict[str, Any]]:
now = float(self._time())
with self._lock:
entry = self._entries.get(auth_id)
if entry is None:
return None
stored_at, value = entry
if now - stored_at >= self._ttl:
self._entries.pop(auth_id, None)
return None
return json.loads(json.dumps(value))
def put(self, auth_id: str, value: Dict[str, Any]) -> None:
with self._lock:
now = float(self._time())
for key, (stored_at, _) in list(self._entries.items()):
if now - stored_at >= self._ttl:
self._entries.pop(key, None)
self._entries.pop(auth_id, None)
while len(self._entries) >= self._max_entries:
self._entries.pop(next(iter(self._entries)))
self._entries[auth_id] = (now, json.loads(json.dumps(value)))
def invalidate(self, auth_id: str) -> None:
with self._lock:
self._entries.pop(auth_id, None)
def clear(self) -> None:
with self._lock:
self._entries.clear()
USAGE_CACHE = UsageCache()
def get_account_usage(
auth_id: str,
owner: Optional[str] = None,
*,
force_refresh: bool = False,
cache: Optional[UsageCache] = None,
) -> Dict[str, Any]:
"""Return normalized usage for exactly one owner-scoped auth session.
The access token is resolved (and refreshed if needed) for that auth
session only. Results are cached per auth id; ``force_refresh`` bypasses
and replaces the cached entry. Raises :class:`ChatGPTUsageUnavailable` on
any read failure; callers must not treat that as an endpoint failure.
"""
cache = USAGE_CACHE if cache is None else cache
auth_id = (auth_id or "").strip()
if not auth_id:
raise ChatGPTSubscriptionAuthNotFound("ChatGPT Subscription account was not found.")
# Authorize even cache hits: cached telemetry must not outlive ownership.
_Auth, SessionLocal, _now = _database_handles()
db = SessionLocal()
try:
if find_owned_auth_session(db, auth_id, owner) is None:
raise ChatGPTSubscriptionAuthNotFound("ChatGPT Subscription account was not found.")
finally:
db.close()
if not force_refresh:
cached = cache.get(auth_id)
if cached is not None:
cached["cached"] = True
return cached
else:
cache.invalidate(auth_id)
try:
creds = resolve_runtime_credentials(auth_id, owner=owner)
except ChatGPTSubscriptionAuthNotFound:
raise
except ChatGPTSubscriptionRateLimited as exc:
raise ChatGPTUsageUnavailable("rate_limited", str(exc), status_code=429) from exc
except ChatGPTSubscriptionReauthRequired as exc:
raise ChatGPTUsageUnavailable("reauth", str(exc), status_code=401) from exc
except ChatGPTSubscriptionError as exc:
raise ChatGPTUsageUnavailable("upstream", str(exc)) from exc
raw = fetch_usage_payload(creds.get("api_key") or "")
normalized = normalize_usage_payload(raw)
normalized["auth_id"] = auth_id
normalized["fetched_at"] = int(time.time())
cache.put(auth_id, normalized)
result = json.loads(json.dumps(normalized))
result["cached"] = False
return result
def to_http_exception(exc: Exception) -> HTTPException:
if isinstance(exc, ChatGPTSubscriptionRateLimited):
return HTTPException(429, str(exc))
if isinstance(exc, (ChatGPTSubscriptionReauthRequired, ChatGPTSubscriptionAuthNotFound)):
return HTTPException(401, f"{exc} Reconnect the provider.")
return HTTPException(502, str(exc))
if isinstance(exc, (ChatGPTSubscriptionError, ValueError)):
return HTTPException(502, str(exc))
return HTTPException(502, "ChatGPT Subscription request failed.")
def build_responses_input(messages: list[dict]) -> list[dict]:
+10 -1
View File
@@ -191,7 +191,16 @@ def build_foreground_route_descriptors(
owner=owner,
)
if selected is None:
selected = resolve_route_descriptor(endpoint_url, model, headers or {}, owner=owner)
if selected_endpoint_id:
# An explicit identity must never be relabelled as another account
# with the same URL/model (or even the same current bearer).
selected = {
"endpoint_id": None,
"endpoint_label": "Selected route",
"endpoint_cost_tracked": endpoint_cost_tracked(endpoint_url),
}
else:
selected = resolve_route_descriptor(endpoint_url, model, headers or {}, owner=owner)
primary = (endpoint_url, model, headers or {})
candidates = [primary]
descriptors = [selected]
+66 -10
View File
@@ -1496,6 +1496,33 @@ def _chatgpt_subscription_instructions(messages: List[Dict]) -> str:
return "You are a helpful AI assistant."
# Provider-native agentic surfaces that must never be sent on the ChatGPT
# Subscription route. ChatGPT provides model inference only; Odysseus is the
# only agent (planning, tool selection/execution, filesystem, shell, browser,
# MCP). Odysseus' own text tool protocol travels inside ``instructions``/``input``.
CHATGPT_FORBIDDEN_PAYLOAD_KEYS = frozenset({
"tools",
"tool_choice",
"parallel_tool_calls",
"web_search",
"web_search_preview",
"file_search",
"computer",
"computer_use",
"computer_use_preview",
"shell",
"local_shell",
"code_interpreter",
"image_generation",
"mcp",
"function",
"functions",
"include",
"previous_response_id",
"background",
})
def _build_chatgpt_responses_payload(
model: str,
messages: List[Dict],
@@ -1503,9 +1530,18 @@ def _build_chatgpt_responses_payload(
max_tokens: int,
*,
stream: bool = False,
tools: Optional[List[Dict]] = None,
**_ignored,
) -> Dict:
"""Build the ChatGPT/Codex Responses request: model inference only.
``tools`` (and any other provider-native tool declaration) is accepted for
signature compatibility with the other transports and deliberately
discarded. See :data:`CHATGPT_FORBIDDEN_PAYLOAD_KEYS`.
"""
from src.chatgpt_subscription import build_responses_input
del tools, _ignored
conversation = [msg for msg in (messages or []) if (msg.get("role") or "") != "system"]
payload: Dict = {
"model": model,
@@ -1519,7 +1555,27 @@ def _build_chatgpt_responses_payload(
# ChatGPT Subscription Codex API does not support max_output_tokens —
# passing it returns HTTP 400 "Unsupported parameter: max_output_tokens".
# Do not include it in the payload.
return payload
return _strip_chatgpt_native_tool_surfaces(payload)
CHATGPT_ALLOWED_PAYLOAD_KEYS = frozenset({
"model", "instructions", "input", "stream", "store", "temperature",
})
def _strip_chatgpt_native_tool_surfaces(payload: Dict) -> Dict:
"""Only explicitly approved inference fields may cross this boundary."""
return {key: value for key, value in payload.items() if key in CHATGPT_ALLOWED_PAYLOAD_KEYS}
def _chatgpt_safe_error(message: str, headers: Dict) -> str:
"""Upstream diagnostics must never echo the request bearer to the UI."""
result = str(message)
for key, value in (headers or {}).items():
if key.lower() in {"authorization", "x-api-key"} and value:
secret = str(value).removeprefix("Bearer ")
result = result.replace(str(value), "[redacted]").replace(secret, "[redacted]")
return result
def _format_chatgpt_subscription_error(status_code: int, text: str) -> str:
@@ -2977,8 +3033,8 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
_clear_host_dead(target_url)
if r.status_code != 200:
raw = (await r.aread()).decode(errors="replace")
friendly = _format_chatgpt_subscription_error(r.status_code, raw)
yield f'event: error\ndata: {json.dumps({"status": r.status_code, "text": friendly, "raw": raw[:500]})}\n\n'
friendly = _format_chatgpt_subscription_error(r.status_code, _chatgpt_safe_error(raw, h))
yield f'event: error\ndata: {json.dumps({"status": r.status_code, "text": friendly})}\n\n'
return
async for line in r.aiter_lines():
if not line:
@@ -3063,13 +3119,13 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
}
text = err.get("message") if isinstance(err, dict) else str(err or "ChatGPT Subscription request failed")
status = _provider_stream_error_status(err, default=400)
yield f'event: error\ndata: {json.dumps({"status": status, "text": text})}\n\n'
yield f'event: error\ndata: {json.dumps({"status": status, "text": _chatgpt_safe_error(text, h)})}\n\n'
return
yield "data: [DONE]\n\n"
except (httpx.ConnectError, httpx.ConnectTimeout) as e:
_cooled = _mark_host_dead(target_url)
_tail = f" — host cooled for {DEAD_HOST_COOLDOWN:.0f}s" if _cooled else " — transient, will retry"
logger.warning(f"ChatGPT Subscription stream connect to {target_url} failed: {e}{_tail}")
logger.warning("ChatGPT Subscription stream connect failed: %s%s", type(e).__name__, _tail)
yield f'event: error\ndata: {json.dumps({"error": f"Cannot reach {_host_key(target_url)}", "status": 503})}\n\n'
except httpx.ReadTimeout:
yield f'event: error\ndata: {json.dumps({"error": "Read timeout", "status": 504})}\n\n'
@@ -3082,8 +3138,8 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
except httpx.NetworkError:
yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n'
except Exception as e:
logger.error(f"ChatGPT Subscription stream error: {e}")
yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n'
logger.error("ChatGPT Subscription stream error: %s", type(e).__name__)
yield f'event: error\ndata: {json.dumps({"error": "ChatGPT Subscription stream failed", "status": 502, "fallback_eligible": False})}\n\n'
return
# ── Native Ollama streaming ──
@@ -3099,7 +3155,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
if r.status_code != 200:
raw = (await r.aread()).decode(errors="replace")
friendly = _format_upstream_error(r.status_code, raw, target_url)
yield f'event: error\ndata: {json.dumps({"status": r.status_code, "text": friendly, "raw": raw[:500]})}\n\n'
yield f'event: error\ndata: {json.dumps({"status": r.status_code, "text": friendly})}\n\n'
return
async for line in r.aiter_lines():
if not line:
@@ -3198,7 +3254,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
if r.status_code != 200:
raw = (await r.aread()).decode(errors="replace")
friendly = _format_upstream_error(r.status_code, raw, target_url)
yield f'event: error\ndata: {json.dumps({"status": r.status_code, "text": friendly, "raw": raw[:500]})}\n\n'
yield f'event: error\ndata: {json.dumps({"status": r.status_code, "text": friendly})}\n\n'
return
async for line in r.aiter_lines():
# SSE allows "data:value" with no space after the colon
@@ -3382,7 +3438,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
if r.status_code != 200:
raw = (await r.aread()).decode(errors="replace")
friendly = _format_upstream_error(r.status_code, raw, target_url)
yield f'event: error\ndata: {json.dumps({"status": r.status_code, "text": friendly, "raw": raw[:500]})}\n\n'
yield f'event: error\ndata: {json.dumps({"status": r.status_code, "text": friendly})}\n\n'
return
first_token_budget = _first_token_timeout(target_url, timeout)