mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-11-06 13:12:27 +01:00
merge(dev): integrate reviewed preview UI changes
This commit is contained in:
Alexandre Teixeira
+125
-13
@@ -569,8 +569,72 @@ async def auto_name_session(session_manager, sess):
|
||||
if not first_msg:
|
||||
return
|
||||
|
||||
from src.owner_identity import is_request_sentinel_owner
|
||||
owner = getattr(sess, "owner", None)
|
||||
invalid_owner = (
|
||||
not isinstance(owner, str) or not owner.strip()
|
||||
or owner != owner.strip() or is_request_sentinel_owner(owner)
|
||||
)
|
||||
if invalid_owner:
|
||||
session_manager.update_session_name(sess.id, fallback_session_title(first_msg))
|
||||
return
|
||||
|
||||
endpoint_url = str(getattr(sess, "endpoint_url", "") or "")
|
||||
model_name = str(getattr(sess, "model", "") or "")
|
||||
|
||||
# The spec ("Ajax") model names its own chats from the user's first
|
||||
# prompt. This function only runs after the first response stream has
|
||||
# settled (auto_name_session_after_stream), so titling does not
|
||||
# contend with the visible answer. Thinking is disabled and the
|
||||
# budget kept small so the model returns a bare title.
|
||||
try:
|
||||
from src.model_profiles import uses_odysseus_progressive_thinking
|
||||
_is_spec_model = bool(model_name) and uses_odysseus_progressive_thinking(model_name)
|
||||
except Exception:
|
||||
_is_spec_model = False
|
||||
if _is_spec_model and endpoint_url:
|
||||
try:
|
||||
from src.database import SessionLocal
|
||||
from src.endpoint_resolver import (build_chat_url, build_headers,
|
||||
resolve_endpoint_runtime, resolve_owner_registered_endpoint)
|
||||
from src.owner_identity import auth_disabled
|
||||
|
||||
owner = getattr(sess, "owner", None)
|
||||
if not owner and not auth_disabled():
|
||||
raise ValueError("Session has no owner")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
endpoint = resolve_owner_registered_endpoint(db, endpoint_url, owner)
|
||||
base, key = resolve_endpoint_runtime(endpoint, owner=owner)
|
||||
title_url = build_chat_url(base)
|
||||
title_headers = build_headers(key, base)
|
||||
finally:
|
||||
db.close()
|
||||
title = await llm_call_async(
|
||||
title_url,
|
||||
model_name,
|
||||
[
|
||||
{"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation \u2014 just the title."},
|
||||
{"role": "user", "content": first_msg},
|
||||
],
|
||||
temperature=0.3,
|
||||
max_tokens=32,
|
||||
headers=title_headers,
|
||||
timeout=20,
|
||||
thinking_mode="off",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("[auto-name] spec title call failed for %s: %s", sess.id, e)
|
||||
title = ""
|
||||
title = str(title or "").strip().strip('"\'').strip()
|
||||
from src.text_helpers import strip_think
|
||||
title = strip_think(title, prose=False, prompt_echo=False)
|
||||
if not title or len(title) >= 80 or "\n" in title:
|
||||
title = fallback_session_title(first_msg)
|
||||
session_manager.update_session_name(sess.id, title)
|
||||
logger.info(f"Auto-named spec session {sess.id}: {title}")
|
||||
return
|
||||
|
||||
if (
|
||||
"ttft" in model_name.lower()
|
||||
or re.search(r":18\d{3}\b", endpoint_url)
|
||||
@@ -581,6 +645,14 @@ async def auto_name_session(session_manager, sess):
|
||||
return
|
||||
|
||||
owner = getattr(sess, "owner", None)
|
||||
from src.owner_identity import auth_disabled
|
||||
if not owner and not auth_disabled():
|
||||
title = fallback_session_title(first_msg)
|
||||
session_manager.update_session_name(sess.id, title)
|
||||
logger.info(f"Auto-named unowned session {sess.id} deterministically: {title}")
|
||||
return
|
||||
|
||||
from src.task_endpoint import resolve_task_endpoint
|
||||
t_url, t_model, t_headers = resolve_task_endpoint(
|
||||
sess.endpoint_url, sess.model, sess.headers, owner=owner
|
||||
)
|
||||
@@ -588,24 +660,64 @@ async def auto_name_session(session_manager, sess):
|
||||
logger.debug("[auto-name] No model provided, skipping")
|
||||
return
|
||||
|
||||
sess_url = getattr(sess, "endpoint_url", None)
|
||||
if t_url and sess_url and t_url == sess_url:
|
||||
from src.database import SessionLocal
|
||||
from src.endpoint_resolver import (
|
||||
build_chat_url,
|
||||
build_headers,
|
||||
resolve_endpoint,
|
||||
resolve_endpoint_runtime,
|
||||
resolve_owner_registered_endpoint,
|
||||
)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
endpoint = resolve_owner_registered_endpoint(db, t_url, owner)
|
||||
base, key = resolve_endpoint_runtime(endpoint, owner=owner)
|
||||
t_url = build_chat_url(base)
|
||||
t_headers = build_headers(key, base)
|
||||
except Exception as e:
|
||||
logger.debug("[auto-name] session fallback endpoint %s unauthorized for %s: %s", t_url, owner, e)
|
||||
t_url, t_model, t_headers = None, None, None
|
||||
for prefix in ("utility", "default"):
|
||||
try:
|
||||
u, m, h = resolve_endpoint(prefix, owner=owner)
|
||||
if u and m:
|
||||
t_url, t_model, t_headers = u, m, h
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if not t_url or not t_model:
|
||||
title = fallback_session_title(first_msg)
|
||||
session_manager.update_session_name(sess.id, title)
|
||||
logger.info(f"Auto-named session {sess.id} deterministically (no authorized model endpoint): {title}")
|
||||
return
|
||||
|
||||
# max_tokens big enough that reasoning models (Minimax M2,
|
||||
# DeepSeek R1, QwQ, etc.) have headroom for <think>…</think>
|
||||
# plus the actual title — 200 used to clip them mid-reasoning
|
||||
# so strip_think left an empty string and no rename happened.
|
||||
# Timeout matches: 60s gives slow local reasoners room to finish.
|
||||
title = await llm_call_async(
|
||||
t_url,
|
||||
t_model,
|
||||
[
|
||||
{"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation — just the title."},
|
||||
{"role": "user", "content": first_msg},
|
||||
],
|
||||
temperature=0.3,
|
||||
max_tokens=64,
|
||||
headers=t_headers,
|
||||
timeout=15,
|
||||
reasoning_effort=effort_for_call(t_url, t_model, owner, sess),
|
||||
)
|
||||
try:
|
||||
title = await llm_call_async(
|
||||
t_url,
|
||||
t_model,
|
||||
[
|
||||
{"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation — just the title."},
|
||||
{"role": "user", "content": first_msg},
|
||||
],
|
||||
temperature=0.3,
|
||||
max_tokens=64,
|
||||
headers=t_headers,
|
||||
timeout=15,
|
||||
reasoning_effort=effort_for_call(t_url, t_model, owner, sess),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("[auto-name] non-spec title call failed for %s: %s", sess.id, e)
|
||||
title = ""
|
||||
|
||||
title = title.strip().strip('"\'').strip()
|
||||
# Strip <think>/<thinking> blocks (closed, dangling, or stray tags)
|
||||
|
||||
+102
-11
@@ -27,11 +27,13 @@ from src.llm_core import (
|
||||
from src.agent_loop import (
|
||||
stream_agent_loop,
|
||||
_configured_model_tool_surface,
|
||||
_configured_model_tool_surface_raw,
|
||||
_local_media_needs_browser_render,
|
||||
_looks_like_workspace_coding_request,
|
||||
)
|
||||
from src.agent_loop import _normalize_ody_qwen_text_artifacts
|
||||
from src import agent_runs
|
||||
from src import shutdown as _shutdown
|
||||
from src.model_context import estimate_tokens
|
||||
from src.context_compactor import (
|
||||
apply_compaction_state,
|
||||
@@ -329,6 +331,7 @@ def _is_external_discovery_request(text: str) -> bool:
|
||||
|
||||
_URL_SCHEME_RE = re.compile(r"https?://", re.I)
|
||||
_PDF_URL_TAIL_RE = re.compile(r"\.pdf\b|/pdf/", re.I)
|
||||
_EDITOR_SESSION_RE = re.compile(r"[A-Za-z0-9@._:-]{1,128}")
|
||||
|
||||
|
||||
def _mentions_pdf_url(value: str) -> bool:
|
||||
@@ -2688,6 +2691,23 @@ def setup_chat_routes(
|
||||
logger.info("chat→agent auto-escalation: explicit private browser workflow")
|
||||
active_doc_id = form_data.get("active_doc_id", "").strip()
|
||||
active_doc_state = form_data.get("active_doc_state", "").strip().casefold()
|
||||
# Scoped photo_editor agent context: the Quick Edit popup (and any
|
||||
# image-edit chat turn) may carry the open image-editor session id and a
|
||||
# canvas snapshot so the editor tool family can drive it in-process.
|
||||
# The id is quoted into a system message, so accept only an
|
||||
# identifier; any other value is treated as no open editor.
|
||||
editor_session = str(
|
||||
form_data.get("editor_session")
|
||||
or form_data.get("editor_session_id")
|
||||
or ""
|
||||
).strip()
|
||||
if not _EDITOR_SESSION_RE.fullmatch(editor_session):
|
||||
editor_session = ""
|
||||
photo_editor_requested = (
|
||||
str(form_data.get("photo_editor") or "").strip().lower()
|
||||
in {"1", "true", "yes", "on"}
|
||||
or bool(editor_session)
|
||||
)
|
||||
logger.info(
|
||||
"[doc-inject] chat_mode=%s, active_doc_id=%r, active_doc_state=%r",
|
||||
chat_mode, active_doc_id, active_doc_state,
|
||||
@@ -3270,7 +3290,34 @@ def setup_chat_routes(
|
||||
active_document=bool(active_doc), workspace=bool(workspace),
|
||||
image_attachment=any(str(a.get('mime') or '').startswith('image/') for a in (ctx.preprocessed.attachment_meta or [])),
|
||||
) if _use_turn_contract else frozenset()
|
||||
if 'image_editing' in _turn_capabilities and ctx.preprocessed.attachment_meta:
|
||||
# Scoped photo_editor family: offered ONLY for an explicit editor
|
||||
# request (Quick Edit popup / editor session) or an image-edit chat
|
||||
# turn carrying an image. When active the turn offers just the editor
|
||||
# tools plus ask_user, never the general tool list.
|
||||
_photo_editor_active = bool(_use_turn_contract) and (
|
||||
photo_editor_requested
|
||||
or (
|
||||
"image_editing" in _turn_capabilities
|
||||
and bool(ctx.preprocessed.attachment_meta)
|
||||
)
|
||||
)
|
||||
if _photo_editor_active:
|
||||
if editor_session:
|
||||
editor_context = {'role': 'system', 'content':
|
||||
'The image editor is open (session ' + editor_session + '). Use run_editor_command, '
|
||||
'plan_edit and get_editor_screenshot to inspect and change the current canvas, and '
|
||||
'ai_edit_image for model-based edits. Apply the user\'s requested edit to the open '
|
||||
'editor and confirm the result with a screenshot.'}
|
||||
else:
|
||||
editor_context = {'role': 'system', 'content':
|
||||
'The user attached an image but no image editor is open, so the editor tools cannot '
|
||||
'reach the canvas. Do not call editor tools. Reply briefly telling the user to open '
|
||||
'the attached image in the editor (Quick Edit) and repeat the request; the editor '
|
||||
'tools will then apply it.'}
|
||||
ctx.messages.insert(0, editor_context)
|
||||
if foreground_policy.enabled:
|
||||
getattr(ctx, 'route_messages', ctx.messages).insert(0, dict(editor_context))
|
||||
elif 'image_editing' in _turn_capabilities and ctx.preprocessed.attachment_meta:
|
||||
image_refs = ['odysseus://attachment/' + str(a['id']) for a in ctx.preprocessed.attachment_meta
|
||||
if a.get('id') and str(a.get('mime') or '').startswith('image/')]
|
||||
if image_refs:
|
||||
@@ -3307,6 +3354,11 @@ def setup_chat_routes(
|
||||
# pasted revision request. This only offers permitted schemas;
|
||||
# it never requires or performs a document mutation.
|
||||
_turn_capabilities = _turn_capabilities | {"documents"}
|
||||
# Scoped photo_editor family (see above): when active the turn offers
|
||||
# just the editor tools plus ask_user, never the general tool list.
|
||||
if _photo_editor_active:
|
||||
_turn_capabilities = frozenset({"photo_editor"})
|
||||
_active_turn_capabilities = _turn_capabilities
|
||||
# Same decision that prepared the turn; it only stamps the contract
|
||||
# inside the agent-contract branch below.
|
||||
_clean_v3_preview = _compact_preview_turn
|
||||
@@ -3474,7 +3526,7 @@ def setup_chat_routes(
|
||||
if not _privs.get("can_use_documents", True):
|
||||
disabled_tools.update({"manage_documents", "create_document", "edit_document", "update_document", "suggest_document"})
|
||||
if not _privs.get("can_generate_images", True):
|
||||
disabled_tools.update({"generate_image", "edit_image"})
|
||||
disabled_tools.update({"generate_image", "edit_image", "ai_edit_image"})
|
||||
if not _privs.get("can_manage_memory", True):
|
||||
disabled_tools.update({"manage_memory", "manage_skills"})
|
||||
if not _privs.get("can_use_research", True):
|
||||
@@ -3556,9 +3608,22 @@ def setup_chat_routes(
|
||||
"document_editor": not {
|
||||
"manage_documents", "create_document", "edit_document", "update_document",
|
||||
}.issubset(disabled_tools),
|
||||
# A photo-editing chat turn can only reach a canvas when the image
|
||||
# editor is actually open. The compact runtime uses this to reply
|
||||
# (open the image in the editor) instead of looping on editor tools
|
||||
# that have no canvas to drive.
|
||||
"photo_editor_open": bool(editor_session),
|
||||
}
|
||||
# Capture permission state before schema selection/reconciliation.
|
||||
# Only deterministic request intent supplies grants, never inventory.
|
||||
# The strict router (spec) surface picks its tools in src/spec_router.py
|
||||
# AFTER authority is built, so its admission is keyed off the surface.
|
||||
try:
|
||||
_spec_surface = _configured_model_tool_surface_raw(
|
||||
getattr(sess, "endpoint_url", ""), getattr(sess, "model", ""), _user
|
||||
) == "spec"
|
||||
except Exception:
|
||||
_spec_surface = False
|
||||
_request_authority = request_authority_for_http(
|
||||
request, message, owner=_user, session_id=session, workspace=workspace,
|
||||
history=_turn_history, policy=tool_policy,
|
||||
@@ -3566,9 +3631,9 @@ def setup_chat_routes(
|
||||
active_document=bool(active_doc),
|
||||
image_attachment=any(str(a.get('mime') or '').startswith('image/')
|
||||
for a in (ctx.preprocessed.attachment_meta or [])),
|
||||
capabilities=({'search_browser'} if (
|
||||
capabilities=(({"photo_editor"} if _photo_editor_active else set()) | ({'search_browser'} if (
|
||||
_explicit_browser_intent or _external_discovery_intent
|
||||
) else ()),
|
||||
) else set())),
|
||||
)
|
||||
if exact_tool_approval is not None:
|
||||
from src.agent_runtime.authority import RequestAuthority
|
||||
@@ -3758,7 +3823,9 @@ def setup_chat_routes(
|
||||
# personal manager; core shell/Web tools are not task
|
||||
# fallbacks. Other turns retain warm and workspace tools.
|
||||
extra_tools=(
|
||||
frozenset()
|
||||
frozenset({"ask_user"})
|
||||
if _active_turn_capabilities == frozenset({"photo_editor"})
|
||||
else frozenset()
|
||||
if _exact_selected_native_chain or _active_turn_capabilities == frozenset({"tasks"}) or (
|
||||
_active_turn_capabilities in (
|
||||
frozenset({"transcription"}), frozenset({"ocr"}),
|
||||
@@ -3960,7 +4027,7 @@ def setup_chat_routes(
|
||||
_heartbeat_counter = 0
|
||||
_last_progress = {}
|
||||
_sent_avg = False
|
||||
while True:
|
||||
while not _shutdown.is_shutting_down():
|
||||
status = research_handler.get_status(session)
|
||||
if not status or status["status"] != "running":
|
||||
break
|
||||
@@ -4222,8 +4289,6 @@ def setup_chat_routes(
|
||||
# indicator, but don't fold them into the saved
|
||||
# reply (mirrors the rewrite path below).
|
||||
if data.get("thinking"):
|
||||
if thinking_mode == "off":
|
||||
continue
|
||||
thinking_response += data["delta"]
|
||||
else:
|
||||
full_response += data["delta"]
|
||||
@@ -4667,8 +4732,6 @@ def setup_chat_routes(
|
||||
# Forward them for the live indicator, but keep
|
||||
# them out of the saved reply (same as chat mode).
|
||||
if data.get("thinking"):
|
||||
if thinking_mode == "off":
|
||||
continue
|
||||
thinking_response += data["delta"]
|
||||
else:
|
||||
full_response = _render_state.content
|
||||
@@ -4852,6 +4915,25 @@ def setup_chat_routes(
|
||||
_terminal_saved = True
|
||||
if _saved_id:
|
||||
yield f'data: {json.dumps(_render_state.message_saved(_saved_id))}\n\n'
|
||||
run_post_response_tasks(
|
||||
sess, session_manager, session, message, _saveable_no_tool_response,
|
||||
_metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
|
||||
incognito=incognito, compare_mode=compare_mode,
|
||||
character_name=ctx.preset.character_name,
|
||||
agent_rounds=_agent_rounds,
|
||||
agent_tool_calls=_agent_tool_calls,
|
||||
skills_manager=skills_manager,
|
||||
owner=_user,
|
||||
extract_skills=(
|
||||
user_requested_agent
|
||||
and not tool_approval_continuation
|
||||
),
|
||||
allow_background_extraction=_post_response_extraction_allowed(
|
||||
tools_blocked=tool_policy.block_all_tool_calls,
|
||||
tool_approval_continuation=tool_approval_continuation,
|
||||
client_runtime_context=client_runtime_context,
|
||||
),
|
||||
)
|
||||
yield f'data: {json.dumps(_metrics_event)}\n\n'
|
||||
except json.JSONDecodeError:
|
||||
yield chunk
|
||||
@@ -4962,6 +5044,8 @@ def setup_chat_routes(
|
||||
raises before reaching a mode-specific finally block."""
|
||||
try:
|
||||
async for chunk in stream_with_save():
|
||||
if _shutdown.is_shutting_down():
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
_active_streams.pop(session, None)
|
||||
@@ -5056,7 +5140,12 @@ def setup_chat_routes(
|
||||
if rec is None:
|
||||
if agent_runs.is_active(session_id):
|
||||
return {"status": "streaming", "detached": True}
|
||||
raise HTTPException(404, "No active stream for this session")
|
||||
# No active stream: report an explicit idle state with HTTP 200.
|
||||
# A 404 here made every session load look like a failed request in
|
||||
# the browser console. The client distinguishes "no stream" by the
|
||||
# body (status != "streaming"), and still tolerates a 404 from an
|
||||
# older server, so this stays backward compatible.
|
||||
return {"status": "idle"}
|
||||
return rec
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
@@ -5155,6 +5244,8 @@ def setup_chat_routes(
|
||||
max_tokens=0,
|
||||
tools=None,
|
||||
):
|
||||
if _shutdown.is_shutting_down():
|
||||
break
|
||||
if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"):
|
||||
try:
|
||||
data = json.loads(chunk[6:])
|
||||
|
||||
@@ -1185,7 +1185,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
|
||||
async def ai_tidy_documents(request: Request) -> Dict[str, Any]:
|
||||
"""Use AI to judge if documents are junk/test/accidental, then delete them.
|
||||
Caches verdicts so previously-reviewed docs are skipped."""
|
||||
from src.task_endpoint import resolve_task_endpoint
|
||||
from src.task_endpoint import resolve_task_endpoint, task_llm_call_async
|
||||
from src.endpoint_resolver import resolve_endpoint
|
||||
from src.llm_core import llm_call_async
|
||||
|
||||
@@ -1228,13 +1228,13 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
|
||||
+ "\n".join(doc_list)
|
||||
)
|
||||
|
||||
response = await llm_call_async(
|
||||
url, model,
|
||||
response = await task_llm_call_async(
|
||||
[{"role": "system", "content": "You classify documents as junk or keep. Respond only with a JSON array."},
|
||||
{"role": "user", "content": prompt}],
|
||||
fallback_url=url, fallback_model=model, fallback_headers=headers,
|
||||
owner=user or None,
|
||||
temperature=0.1,
|
||||
max_tokens=200,
|
||||
headers=headers,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+206
-63
@@ -43,6 +43,7 @@ from starlette.concurrency import run_in_threadpool
|
||||
from fastapi import APIRouter, Query, UploadFile, File, BackgroundTasks, HTTPException, Depends, Request
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
from src.constants import DATA_DIR
|
||||
from src import shutdown as _shutdown
|
||||
from src.path_confinement import confine
|
||||
|
||||
from src.llm_core import llm_call_async
|
||||
@@ -1826,20 +1827,75 @@ def setup_email_routes():
|
||||
# a cached message body.
|
||||
return (account_id or "", folder, str(uid), owner)
|
||||
|
||||
_LIST_STALE = {}
|
||||
_LIST_LOCK = _threading.RLock()
|
||||
_LIST_CACHE_LOCK = _LIST_LOCK
|
||||
# Invalidation fences reads already in flight, including background refresh.
|
||||
_LIST_EPOCH = [0]
|
||||
_LIST_CACHE_GENERATION = _LIST_EPOCH
|
||||
|
||||
def _list_cache_get(key):
|
||||
v = _LIST_CACHE.get(key)
|
||||
if not v: return None
|
||||
if v[0] < _time.monotonic():
|
||||
_LIST_CACHE.pop(key, None)
|
||||
return None
|
||||
return v[1]
|
||||
with _LIST_LOCK:
|
||||
v = _LIST_CACHE.get(key)
|
||||
if not v: return None
|
||||
if v[0] < _time.monotonic():
|
||||
_LIST_CACHE.pop(key, None)
|
||||
return None
|
||||
return v[1]
|
||||
|
||||
def _list_stale_get(key):
|
||||
with _LIST_LOCK:
|
||||
return _LIST_STALE.get(key)
|
||||
|
||||
def _list_stale_put(key, value):
|
||||
_LIST_STALE[key] = value
|
||||
if len(_LIST_STALE) > 96:
|
||||
for k in list(_LIST_STALE.keys())[:-48]:
|
||||
_LIST_STALE.pop(k, None)
|
||||
|
||||
_LIST_REFRESHING = set()
|
||||
|
||||
def _list_cache_put(key, value):
|
||||
_LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value)
|
||||
# Cap size
|
||||
if len(_LIST_CACHE) > 64:
|
||||
for k in list(_LIST_CACHE.keys())[:-32]:
|
||||
_LIST_CACHE.pop(k, None)
|
||||
with _LIST_LOCK:
|
||||
_LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value)
|
||||
# Also snapshot the last-good response for stale-while-revalidate.
|
||||
_list_stale_put(key, value)
|
||||
if len(_LIST_CACHE) > 64:
|
||||
for k in list(_LIST_CACHE.keys())[:-32]:
|
||||
_LIST_CACHE.pop(k, None)
|
||||
|
||||
def _schedule_list_refresh(sync_args, cache_key, folder, account_id, owner):
|
||||
"""Refresh a list snapshot off the event loop, at most once per key.
|
||||
|
||||
Runs the blocking IMAP fetch in a daemon thread, then repopulates both
|
||||
the TTL cache and the stale snapshot so the next list view is warm.
|
||||
"""
|
||||
with _LIST_LOCK:
|
||||
if cache_key in _LIST_REFRESHING:
|
||||
return
|
||||
_LIST_REFRESHING.add(cache_key)
|
||||
epoch = _LIST_EPOCH[0]
|
||||
|
||||
def _run():
|
||||
try:
|
||||
refreshed = _list_emails_sync(*sync_args)
|
||||
if refreshed and not refreshed.get("error"):
|
||||
with _LIST_LOCK:
|
||||
if epoch != _LIST_EPOCH[0]:
|
||||
return
|
||||
_list_cache_put(cache_key, refreshed)
|
||||
try:
|
||||
_schedule_recent_email_warm(
|
||||
refreshed.get("emails") or [], folder, account_id, owner)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc: # pragma: no cover - background best effort
|
||||
logger.debug("email list background refresh failed: %s", exc)
|
||||
finally:
|
||||
with _LIST_LOCK:
|
||||
_LIST_REFRESHING.discard(cache_key)
|
||||
|
||||
_threading.Thread(target=_run, daemon=True, name="email-list-refresh").start()
|
||||
|
||||
def _folder_cache_get(account_id, owner):
|
||||
key = (account_id or "", owner or "")
|
||||
@@ -1874,37 +1930,74 @@ def setup_email_routes():
|
||||
hook(account_id, folder)
|
||||
except Exception:
|
||||
pass
|
||||
if account_id is None and folder is None:
|
||||
_LIST_CACHE.clear()
|
||||
return
|
||||
for k in list(_LIST_CACHE.keys()):
|
||||
k_acct = k[0] if len(k) > 0 else ""
|
||||
k_folder = k[1] if len(k) > 1 else ""
|
||||
if (account_id is None or k_acct == (account_id or "")) and \
|
||||
(folder is None or k_folder == folder):
|
||||
_LIST_CACHE.pop(k, None)
|
||||
with _LIST_LOCK:
|
||||
_LIST_EPOCH[0] += 1
|
||||
if account_id is None and folder is None:
|
||||
_LIST_CACHE.clear()
|
||||
_LIST_STALE.clear()
|
||||
return
|
||||
for cache in (_LIST_CACHE, _LIST_STALE):
|
||||
for k in list(cache.keys()):
|
||||
k_acct = k[0] if len(k) > 0 else ""
|
||||
k_folder = k[1] if len(k) > 1 else ""
|
||||
if (account_id is None or k_acct == (account_id or "")) and \
|
||||
(folder is None or k_folder == folder):
|
||||
cache.pop(k, None)
|
||||
|
||||
def _index_update_flags(owner, account_id, folder, uid, flag, add):
|
||||
with _LIST_LOCK:
|
||||
_email_index_update_flags(owner, account_id, folder, uid, flag, add)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
|
||||
def _index_delete(owner, account_id, folder, uid):
|
||||
with _LIST_LOCK:
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
|
||||
def _update_list_cache_seen(account_id, folder, uid, seen: bool):
|
||||
uid_s = str(uid)
|
||||
for key, (expires_at, value) in list(_LIST_CACHE.items()):
|
||||
if key[0] != (account_id or "") or key[1] != folder:
|
||||
continue
|
||||
emails = list((value or {}).get("emails") or [])
|
||||
changed = False
|
||||
if seen and key[2] == "unread":
|
||||
kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s]
|
||||
if len(kept) != len(emails):
|
||||
value = dict(value)
|
||||
value["emails"] = kept
|
||||
value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept)))
|
||||
changed = True
|
||||
else:
|
||||
for e in emails:
|
||||
if str((e or {}).get("uid") or "") == uid_s:
|
||||
e["is_read"] = bool(seen)
|
||||
with _LIST_LOCK:
|
||||
for key, (expires_at, value) in list(_LIST_CACHE.items()):
|
||||
if key[0] != (account_id or "") or key[1] != folder:
|
||||
continue
|
||||
emails = list((value or {}).get("emails") or [])
|
||||
changed = False
|
||||
if seen and key[2] == "unread":
|
||||
kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s]
|
||||
if len(kept) != len(emails):
|
||||
value = dict(value)
|
||||
value["emails"] = kept
|
||||
value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept)))
|
||||
changed = True
|
||||
if changed:
|
||||
_LIST_CACHE[key] = (expires_at, value)
|
||||
else:
|
||||
for e in emails:
|
||||
if str((e or {}).get("uid") or "") == uid_s:
|
||||
e["is_read"] = bool(seen)
|
||||
changed = True
|
||||
if changed:
|
||||
_LIST_CACHE[key] = (expires_at, value)
|
||||
if key in _LIST_STALE:
|
||||
_LIST_STALE[key] = value
|
||||
|
||||
for key, value in list(_LIST_STALE.items()):
|
||||
if key[0] != (account_id or "") or key[1] != folder:
|
||||
continue
|
||||
emails = list((value or {}).get("emails") or [])
|
||||
changed = False
|
||||
if seen and key[2] == "unread":
|
||||
kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s]
|
||||
if len(kept) != len(emails):
|
||||
value = dict(value)
|
||||
value["emails"] = kept
|
||||
value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept)))
|
||||
changed = True
|
||||
else:
|
||||
for e in emails:
|
||||
if str((e or {}).get("uid") or "") == uid_s:
|
||||
e["is_read"] = bool(seen)
|
||||
changed = True
|
||||
if changed:
|
||||
_LIST_STALE[key] = value
|
||||
|
||||
def _read_cache_get(key):
|
||||
v = _READ_CACHE.get(key)
|
||||
@@ -2284,6 +2377,8 @@ def setup_email_routes():
|
||||
SECURITY: `owner` is propagated so when `account_id` is missing,
|
||||
the fallback config lookup is scoped to this user's accounts only.
|
||||
"""
|
||||
with _LIST_CACHE_LOCK:
|
||||
index_generation = _LIST_CACHE_GENERATION[0]
|
||||
conn = None
|
||||
conn_ok = False
|
||||
try:
|
||||
@@ -2578,7 +2673,9 @@ def setup_email_routes():
|
||||
if parsed:
|
||||
fetched_emails.append(parsed)
|
||||
cached_rows[parsed["uid"]] = parsed
|
||||
_email_index_upsert(owner, account_id, folder, fetched_emails)
|
||||
with _LIST_CACHE_LOCK:
|
||||
if index_generation == _LIST_CACHE_GENERATION[0]:
|
||||
_email_index_upsert(owner, account_id, folder, fetched_emails)
|
||||
for uid_num in uid_order:
|
||||
row = cached_rows.get(uid_num)
|
||||
if not row:
|
||||
@@ -2867,6 +2964,48 @@ def setup_email_routes():
|
||||
owner, account_id or "", folder, filter, limit, offset, elapsed_ms,
|
||||
)
|
||||
return cached
|
||||
# Stale-while-revalidate: never block a plain list view on IMAP while
|
||||
# we already hold a recent snapshot or a durable index. Serve it now
|
||||
# with a `stale` marker and refresh in the background so the next view
|
||||
# is warm. Blocking IMAP only remains for a genuine cold start with no
|
||||
# snapshot and no indexed rows (e.g. a brand-new account/folder).
|
||||
if not manual_refresh and not from_addr and not has_attachments and not (date_from or date_to):
|
||||
snapshot = _list_stale_get(ck)
|
||||
if snapshot is None:
|
||||
try:
|
||||
indexed_emails, indexed_total, indexed_at = _email_index_list(
|
||||
owner, account_id, folder, filter, limit, offset, False,
|
||||
)
|
||||
except Exception:
|
||||
indexed_emails, indexed_total, indexed_at = [], 0, None
|
||||
if indexed_total:
|
||||
snapshot = {
|
||||
"emails": indexed_emails,
|
||||
"total": indexed_total,
|
||||
"folder": folder,
|
||||
"offset": offset,
|
||||
"sync": {
|
||||
"source": "index",
|
||||
"indexed": indexed_total,
|
||||
"updated_at": indexed_at,
|
||||
},
|
||||
}
|
||||
if snapshot is not None:
|
||||
payload = dict(snapshot)
|
||||
payload["emails"] = list(payload.get("emails") or [])
|
||||
_hide_unlinked_calendar_tags(payload["emails"])
|
||||
sync_meta = dict(payload.get("sync") or {})
|
||||
sync_meta["source"] = "cache_stale" if _list_stale_get(ck) is not None else "index"
|
||||
sync_meta["stale"] = True
|
||||
payload["sync"] = sync_meta
|
||||
_schedule_list_refresh(
|
||||
(folder, limit, offset, filter, account_id, from_addr,
|
||||
bool(has_attachments), owner, True, date_from or "", date_to or ""),
|
||||
ck, folder, account_id, owner,
|
||||
)
|
||||
return payload
|
||||
with _LIST_CACHE_LOCK:
|
||||
cache_generation = _LIST_CACHE_GENERATION[0]
|
||||
result = await _asyncio.to_thread(
|
||||
_list_emails_sync, folder, limit, offset, filter, account_id, from_addr,
|
||||
bool(has_attachments), owner, manual_refresh, date_from or "", date_to or "",
|
||||
@@ -2875,7 +3014,9 @@ def setup_email_routes():
|
||||
if offset == 0 and not from_addr and not has_attachments and filter in ("all", "unread", "unanswered", "undone"):
|
||||
_record_email_received_events(owner, account_id, folder, result.get("emails") or [])
|
||||
_schedule_recent_email_warm(result.get("emails") or [], folder, account_id, owner)
|
||||
_list_cache_put(ck, result)
|
||||
with _LIST_CACHE_LOCK:
|
||||
if cache_generation == _LIST_CACHE_GENERATION[0]:
|
||||
_list_cache_put(ck, result)
|
||||
elapsed_ms = int((_time.monotonic() - started_at) * 1000)
|
||||
if elapsed_ms > 1500:
|
||||
logger.warning(
|
||||
@@ -3223,7 +3364,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
moved = _move_email_message(conn, uid, "Junk", role="junk")
|
||||
if moved:
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id)
|
||||
except Exception:
|
||||
logger.debug("unsubscribe move-to-spam skipped", exc_info=True)
|
||||
@@ -3240,7 +3381,7 @@ def setup_email_routes():
|
||||
if deleted:
|
||||
conn.expunge()
|
||||
if deleted:
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id)
|
||||
except Exception as exc:
|
||||
delete_error = "Email was unsubscribed but could not be moved to Trash"
|
||||
@@ -3322,7 +3463,7 @@ def setup_email_routes():
|
||||
if moved:
|
||||
changed += 1
|
||||
cleaned_uids.append(uid)
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
else:
|
||||
failed += 1
|
||||
except Exception:
|
||||
@@ -3411,6 +3552,8 @@ def setup_email_routes():
|
||||
folder (Gmail does), we transparently swap to All Mail so the
|
||||
search surfaces archived / labelled emails too. Plain IMAP
|
||||
accounts fall back to whatever folder the caller specified."""
|
||||
with _LIST_CACHE_LOCK:
|
||||
index_generation = _LIST_CACHE_GENERATION[0]
|
||||
if not q or len(q) < 2:
|
||||
return {"emails": [], "total": 0, "query": q}
|
||||
# CRLF in q would terminate the IMAP command early — reject defensively.
|
||||
@@ -3554,7 +3697,9 @@ def setup_email_routes():
|
||||
except Exception as e:
|
||||
logger.warning(f"Error parsing search result {uid}: {e}")
|
||||
continue
|
||||
_email_index_upsert(owner, account_id, effective_folder, fetched_emails)
|
||||
with _LIST_CACHE_LOCK:
|
||||
if index_generation == _LIST_CACHE_GENERATION[0]:
|
||||
_email_index_upsert(owner, account_id, effective_folder, fetched_emails)
|
||||
|
||||
return {
|
||||
"emails": emails,
|
||||
@@ -3691,8 +3836,7 @@ def setup_email_routes():
|
||||
# accepted it, so the index and list cache cannot drift ahead of
|
||||
# the mailbox.
|
||||
if mark_seen and not mark_seen_failed:
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
_update_list_cache_seen(account_id, folder, uid, True)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
|
||||
related_attachments = []
|
||||
if full and not _has_visible_attachments(msg):
|
||||
@@ -3831,8 +3975,7 @@ def setup_email_routes():
|
||||
status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "(\\Seen)")
|
||||
if status != "OK":
|
||||
return False
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
_update_list_cache_seen(account_id, folder, uid, True)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"mark-seen after cached read failed uid={uid}: {e}")
|
||||
@@ -5064,7 +5207,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Seen", add=False):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", False)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Seen", False)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5084,7 +5227,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Flagged", add=bool(on)):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on))
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on))
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True, "flagged": bool(on)}
|
||||
except Exception as e:
|
||||
@@ -5102,7 +5245,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Seen", add=True):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5122,7 +5265,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _move_email_message(conn, uid, "Archive", role="archive"):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5153,7 +5296,7 @@ def setup_email_routes():
|
||||
# click or by another mailbox client; it is already gone
|
||||
# from the requested folder, so do not trap the UI on a
|
||||
# permanent “Email not found” error.
|
||||
_email_index_delete(owner, account_id, folder, str(uid))
|
||||
_index_delete(owner, account_id, folder, str(uid))
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True, "already_deleted": True}
|
||||
trash_folder = _resolve_mail_folder(conn, "Trash", "trash")
|
||||
@@ -5171,9 +5314,9 @@ def setup_email_routes():
|
||||
if not _move_email_message(conn, resolved_uid, trash_folder, role="trash"):
|
||||
logger.warning("Email delete Trash move failed uid=%s resolved_uid=%s folder=%s trash=%s", uid, resolved_uid, folder, trash_folder)
|
||||
return {"success": False, "error": "Could not move email to Trash"}
|
||||
_email_index_delete(owner, account_id, folder, resolved_uid)
|
||||
_index_delete(owner, account_id, folder, resolved_uid)
|
||||
if resolved_uid != str(uid):
|
||||
_email_index_delete(owner, account_id, folder, str(uid))
|
||||
_index_delete(owner, account_id, folder, str(uid))
|
||||
_invalidate_list_cache(account_id)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5189,7 +5332,7 @@ def setup_email_routes():
|
||||
if not _store_email_flag(conn, uid, "\\Deleted", add=True):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
conn.expunge()
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5257,7 +5400,7 @@ def setup_email_routes():
|
||||
pending_expunge.append(uid)
|
||||
continue
|
||||
deleted_uids.append(uid)
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
except Exception:
|
||||
failed_uids.append(uid)
|
||||
logger.debug("Bulk email delete failed for uid=%s", uid, exc_info=True)
|
||||
@@ -5279,7 +5422,7 @@ def setup_email_routes():
|
||||
if expunge_ok:
|
||||
for uid in pending_expunge:
|
||||
deleted_uids.append(uid)
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
else:
|
||||
# Prioritize preservation over cleanup: unflag \Deleted from our
|
||||
# copied messages so future EXPUNGE operations do not destroy them.
|
||||
@@ -5398,7 +5541,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _move_email_message(conn, uid, dest):
|
||||
return {"success": False, "error": f"Failed to move to {dest}"}
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5484,7 +5627,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Answered", add=True):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Answered", True)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Answered", True)
|
||||
_clear_done_response_tags(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
@@ -5500,7 +5643,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Answered", add=False):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Answered", False)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Answered", False)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -6167,7 +6310,7 @@ def setup_email_routes():
|
||||
try:
|
||||
st, _sel = imap.select(_q(_source_folder), readonly=False)
|
||||
if st == "OK" and _store_email_flag(imap, _source_uid, "\\Answered", add=True):
|
||||
_email_index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True)
|
||||
_index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True)
|
||||
_clear_done_response_tags(owner, _account_id, _source_folder, _source_uid)
|
||||
_invalidate_list_cache(_account_id, _source_folder)
|
||||
logger.info(f"Marked source UID {_source_uid} as \\Answered in {_source_folder}")
|
||||
@@ -6717,7 +6860,7 @@ def setup_email_routes():
|
||||
await queue.put({'type': 'result', **result})
|
||||
task = asyncio.create_task(run())
|
||||
try:
|
||||
while True:
|
||||
while not _shutdown.is_shutting_down():
|
||||
item = await queue.get()
|
||||
yield 'data: ' + json.dumps(item) + '\n\n'
|
||||
if item.get('type') == 'result':
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,7 @@ import os
|
||||
import uuid
|
||||
import urllib.parse
|
||||
import html
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
from fastapi import APIRouter, Form, HTTPException, Request
|
||||
from fastapi.responses import RedirectResponse, HTMLResponse
|
||||
@@ -12,7 +13,7 @@ import logging
|
||||
import httpx
|
||||
|
||||
from core.database import McpServer, SessionLocal
|
||||
from core.middleware import require_admin
|
||||
from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN, require_admin
|
||||
from src.constants import DATA_DIR, MCP_OAUTH_DIR
|
||||
from src.mcp_manager import McpManager
|
||||
|
||||
@@ -424,6 +425,11 @@ def setup_mcp_routes(mcp_manager: McpManager):
|
||||
Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]}
|
||||
"""
|
||||
require_admin(request)
|
||||
internal_token = request.headers.get(INTERNAL_TOOL_HEADER)
|
||||
if internal_token and secrets.compare_digest(internal_token, INTERNAL_TOOL_TOKEN):
|
||||
# Same boundary as POST /api/tools: an agent turn must not be able
|
||||
# to re-enable MCP tools an admin switched off.
|
||||
raise HTTPException(403, "Change MCP tool policy in Settings > Integrations, not from chat")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
srv = db.query(McpServer).filter(McpServer.id == server_id).first()
|
||||
|
||||
+29
-7
@@ -7,6 +7,7 @@ import json
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import socket
|
||||
import secrets
|
||||
import time as _time
|
||||
import logging
|
||||
import threading
|
||||
@@ -473,7 +474,7 @@ def _truthy(value: str | None) -> bool:
|
||||
|
||||
_ENDPOINT_KINDS = {"auto", "local", "api", "proxy"}
|
||||
_REFRESH_MODES = {"auto", "manual", "disabled"}
|
||||
_MODEL_TOOL_MODES = {"none", "compact", "full"}
|
||||
_MODEL_TOOL_MODES = {"none", "compact", "full", "spec"}
|
||||
_MODEL_TOOL_MODE_ALIASES = {
|
||||
"regular": "full",
|
||||
"odysseus_compact": "compact",
|
||||
@@ -553,10 +554,10 @@ def _endpoint_refresh_timeout(ep: Any, category: str) -> float:
|
||||
val = 0
|
||||
if val > 0:
|
||||
return float(max(1, min(60, val)))
|
||||
# llama.cpp and other local OpenAI-compatible servers can block briefly
|
||||
# while warming/loading. A 2s local timeout makes working endpoints flicker
|
||||
# offline before /v1/models is ready.
|
||||
return 10.0 if category == "local" else 2.0
|
||||
# Keep the background refresh short so a single dead endpoint cannot stall
|
||||
# the model picker. A timed-out refresh never clears a non-empty model
|
||||
# cache, so a briefly-warming local server is simply retried next round.
|
||||
return 2.0
|
||||
|
||||
|
||||
def _manual_refresh_timeout(ep: Any, category: str, requested: Any = None) -> float:
|
||||
@@ -1511,6 +1512,9 @@ def setup_model_routes(model_discovery):
|
||||
_refresh_inflight = {"v": False, "done": None} # coarse single-flight guard
|
||||
_REFRESH_FAILURE_BASE = 300.0
|
||||
_REFRESH_FAILURE_MAX = 3600.0
|
||||
# Even a user-triggered refresh must not re-probe an endpoint that just
|
||||
# failed/timed out on every click; keep a short cooldown.
|
||||
_REFRESH_FAILURE_MIN = 60.0
|
||||
|
||||
def _refresh_key(base: str, api_key: Optional[str], provider_auth_id: Optional[str] = None) -> str:
|
||||
# Session-backed endpoints carry no static key; include their auth id
|
||||
@@ -1558,14 +1562,20 @@ def setup_model_routes(model_discovery):
|
||||
if mode in ("manual", "disabled") and not force:
|
||||
return False, info
|
||||
fails = int(state.get("fail_count") or 0)
|
||||
if fails and not force:
|
||||
if fails:
|
||||
last_failure = float(state.get("last_failure") or 0.0)
|
||||
empty_local = (
|
||||
not cached
|
||||
and category == "local"
|
||||
and str(getattr(ep, "id", "") or "").startswith("local-")
|
||||
)
|
||||
if now - last_failure < _failure_delay(fails, empty_local=empty_local):
|
||||
delay = _failure_delay(fails, empty_local=empty_local)
|
||||
if force:
|
||||
# A user-triggered refresh gets a short cooldown instead of the
|
||||
# long background backoff, but still skips a just-failed
|
||||
# endpoint so repeated clicks cannot stall the picker.
|
||||
delay = min(delay, _REFRESH_FAILURE_MIN)
|
||||
if now - last_failure < delay:
|
||||
return False, info
|
||||
if cached and not force:
|
||||
interval = _endpoint_refresh_interval(ep, category)
|
||||
@@ -3082,6 +3092,12 @@ def setup_model_routes(model_discovery):
|
||||
def update_tools(body: ToolsUpdate, request: Request):
|
||||
"""Update which tools are disabled."""
|
||||
require_admin(request)
|
||||
from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN
|
||||
internal_token = request.headers.get(INTERNAL_TOOL_HEADER)
|
||||
if internal_token and secrets.compare_digest(internal_token, INTERNAL_TOOL_TOKEN):
|
||||
# Owner impersonation can carry an admin username, so check the
|
||||
# authenticated transport rather than request.state.current_user.
|
||||
raise HTTPException(403, "Change tool policy in Settings > Agent Tools, not from chat")
|
||||
settings = _load_settings()
|
||||
settings["disabled_tools"] = body.disabled
|
||||
_save_settings(settings)
|
||||
@@ -3089,4 +3105,10 @@ def setup_model_routes(model_discovery):
|
||||
|
||||
router._should_refresh_endpoint = _should_refresh_endpoint
|
||||
router._search_endpoint_catalog = search_endpoint_catalog
|
||||
# Exposed for tests: the in-memory backoff state, its key function, and the
|
||||
# background refresh driver (probes endpoints in parallel with the short
|
||||
# per-endpoint timeout).
|
||||
router._refresh_state = _refresh_state
|
||||
router._refresh_key = _refresh_key
|
||||
router._refresh_caches_bg = _refresh_caches_bg
|
||||
return router
|
||||
|
||||
@@ -17,6 +17,7 @@ from src.endpoint_resolver import resolve_endpoint
|
||||
from src.auth_helpers import _auth_disabled, get_current_user
|
||||
from src.owner_identity import REQUEST_SENTINEL_OWNERS
|
||||
from src.constants import DEEP_RESEARCH_DIR
|
||||
from src import shutdown as _shutdown
|
||||
|
||||
_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
|
||||
|
||||
@@ -285,11 +286,17 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
|
||||
async def research_status(session_id: str, request: Request):
|
||||
user = _require_user(request)
|
||||
_validate_session_id(session_id)
|
||||
# No in-flight or persisted research for this session is a normal state
|
||||
# (most sessions have none), not an error. Return HTTP 200 with an
|
||||
# explicit empty status so loading a session does not log a spurious
|
||||
# 404; every client treats a non-"running" status as idle anyway.
|
||||
# Research owned by ANOTHER user answers identically, so the response
|
||||
# does not reveal whether that research exists.
|
||||
if not _owns_in_memory(session_id, user):
|
||||
raise HTTPException(404, "No research found for this session")
|
||||
return {"status": "none"}
|
||||
status = research_handler.get_status(session_id)
|
||||
if status is None:
|
||||
raise HTTPException(404, "No research found for this session")
|
||||
return {"status": "none"}
|
||||
try:
|
||||
source_state = research_handler.get_source_state(session_id)
|
||||
if isinstance(source_state, str) and source_state:
|
||||
@@ -657,7 +664,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
|
||||
raise HTTPException(404, "No research found for this session")
|
||||
async def _generate():
|
||||
last_payload = None
|
||||
while True:
|
||||
while not _shutdown.is_shutting_down():
|
||||
status = research_handler.get_status(session_id)
|
||||
if status is None:
|
||||
yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
|
||||
|
||||
@@ -1320,7 +1320,7 @@ def setup_session_routes(
|
||||
after Phase 1 — used by the "Tidy (no AI)" UI affordance so
|
||||
users can clean junk without spending tokens.
|
||||
"""
|
||||
from src.llm_core import llm_call
|
||||
from src.llm_core import llm_call, llm_call_with_fallback
|
||||
from src.utility_effort import effort_for_call
|
||||
user = effective_user(request)
|
||||
single_user_mode = not user and _auth_disabled()
|
||||
@@ -1467,7 +1467,7 @@ def setup_session_routes(
|
||||
return {"status": "skipped", "reason": "No unfiled sessions to sort"}
|
||||
|
||||
# Pick an endpoint — prefer admin-configured task endpoint
|
||||
from src.task_endpoint import resolve_task_endpoint
|
||||
from src.task_endpoint import resolve_task_endpoint, resolve_task_candidates
|
||||
url, model, headers = resolve_task_endpoint(owner=user)
|
||||
if not url:
|
||||
url, model, headers = _pick_endpoint_for_sort(owner=user)
|
||||
@@ -1493,9 +1493,12 @@ def setup_session_routes(
|
||||
# 16384 (was 4096): with many chats the folder JSON is large, and a
|
||||
# reasoning model spends tokens thinking first — 4096 truncated the
|
||||
# JSON mid-output, so it never parsed ("invalid JSON for auto-sort").
|
||||
raw = llm_call(url, model, [{"role": "user", "content": prompt}],
|
||||
temperature=0.3, max_tokens=16384, headers=headers, timeout=120,
|
||||
reasoning_effort=effort_for_call(url, model, user))
|
||||
raw = llm_call_with_fallback(
|
||||
resolve_task_candidates(fallback_url=url, fallback_model=model,
|
||||
fallback_headers=headers, owner=user),
|
||||
[{"role": "user", "content": prompt}],
|
||||
temperature=0.3, max_tokens=16384, timeout=120,
|
||||
reasoning_effort=effort_for_call(url, model, user))
|
||||
logger.info(f"Auto-sort raw response ({len(raw)} chars): {raw[:300]}")
|
||||
# Extract JSON from response — handle markdown fences, leading text,
|
||||
# reasoning-model <think> blocks, and trailing commas.
|
||||
|
||||
@@ -27,6 +27,7 @@ from src.host_docker_access import (
|
||||
from src.optional_deps import prepare_optional_dependency_import
|
||||
from src.auth_helpers import _auth_disabled
|
||||
from src import process_lifecycle
|
||||
from src import shutdown as _shutdown
|
||||
|
||||
# POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
|
||||
# on Windows, so importing them unconditionally crashed app startup there
|
||||
@@ -1025,7 +1026,7 @@ async def _generate_tmux(cmd: str, request: Request):
|
||||
lines_sent = 0
|
||||
exit_code = None
|
||||
|
||||
while True:
|
||||
while not _shutdown.is_shutting_down():
|
||||
# Check client disconnect
|
||||
if await request.is_disconnected():
|
||||
# tmux keeps running — that's the whole point
|
||||
@@ -1142,7 +1143,7 @@ async def _generate_win_detached(cmd: str, request: Request):
|
||||
|
||||
lines_sent = 0
|
||||
exit_code = None
|
||||
while True:
|
||||
while not _shutdown.is_shutting_down():
|
||||
if await request.is_disconnected():
|
||||
yield f"data: {json.dumps({'stream': 'stdout', 'data': f'Disconnected. Background job {session_id} continues running.'})}\n\n"
|
||||
return
|
||||
@@ -1267,7 +1268,7 @@ def setup_shell_routes() -> APIRouter:
|
||||
"""Read chunks, split on \\n or \\r for progress bar support."""
|
||||
try:
|
||||
buf = b""
|
||||
while True:
|
||||
while not _shutdown.is_shutting_down():
|
||||
chunk = await stream.read(4096)
|
||||
if not chunk:
|
||||
if buf:
|
||||
|
||||
+39
-26
@@ -435,11 +435,11 @@ def setup_upload_routes(upload_handler):
|
||||
logger.error(f"Failed to get upload stats: {e}")
|
||||
raise HTTPException(500, "Failed to get upload statistics")
|
||||
|
||||
@router.get("/{file_id}")
|
||||
@router.api_route("/{file_id}", methods=["GET", "HEAD"])
|
||||
async def download_file(request: Request, file_id: str, thumb: int = 0):
|
||||
"""Serve an uploaded file by its ID. `?thumb=1` returns a small cached
|
||||
JPEG thumbnail for images (used by chat attachment previews) so the
|
||||
client isn't downloading the full-resolution photo just to show it tiny."""
|
||||
"""Serve an uploaded file by its ID or probe via HEAD. `?thumb=1` returns
|
||||
a small cached JPEG thumbnail for images (used by chat attachment previews)
|
||||
so the client isn't downloading the full-resolution photo just to show it tiny."""
|
||||
if not upload_handler.validate_upload_id(file_id):
|
||||
raise HTTPException(400, "Invalid file ID")
|
||||
import mimetypes as _mt
|
||||
@@ -458,29 +458,42 @@ def setup_upload_routes(upload_handler):
|
||||
mime = (info or {}).get("mime") or _mt.guess_type(path)[0] or "application/octet-stream"
|
||||
from fastapi.responses import FileResponse
|
||||
# Downscaled thumbnail for image previews — generated once and cached.
|
||||
# Skip generation during HEAD probe to avoid loading/processing whole image.
|
||||
if thumb and mime.startswith("image/"):
|
||||
try:
|
||||
from PIL import Image, ImageOps
|
||||
thumb_dir = os.path.join(_upload_root(), ".thumbs")
|
||||
os.makedirs(thumb_dir, exist_ok=True)
|
||||
thumb_path = os.path.join(thumb_dir, file_id + ".jpg")
|
||||
if (not os.path.exists(thumb_path)
|
||||
or os.path.getmtime(thumb_path) < os.path.getmtime(path)):
|
||||
im = Image.open(path)
|
||||
# iPhone / camera JPEGs encode rotation in EXIF rather than
|
||||
# the pixel data. Browsers honour that on the original via
|
||||
# image-orientation:from-image, but PIL strips EXIF when it
|
||||
# saves the JPEG thumb, leaving the pixels sideways. Bake
|
||||
# the rotation into the pixels before thumbnailing.
|
||||
im = ImageOps.exif_transpose(im)
|
||||
im.thumbnail((320, 320))
|
||||
if im.mode not in ("RGB", "L"):
|
||||
im = im.convert("RGB")
|
||||
im.save(thumb_path, "JPEG", quality=80)
|
||||
return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS)
|
||||
except Exception as e:
|
||||
logger.warning(f"Thumbnail generation failed for {file_id}: {e}")
|
||||
# Fall through to the full image.
|
||||
thumb_dir = os.path.join(_upload_root(), ".thumbs")
|
||||
thumb_path = os.path.join(thumb_dir, file_id + ".jpg")
|
||||
if request.method == "HEAD":
|
||||
if (os.path.isfile(thumb_path)
|
||||
and os.path.getmtime(thumb_path) >= os.path.getmtime(path)):
|
||||
return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS)
|
||||
# GET may generate a JPEG or fall back to the original on failure.
|
||||
# Neither representation is known without decoding the image. HEAD
|
||||
# must not advertise the original/stale file's headers as a thumb.
|
||||
from fastapi.responses import Response
|
||||
response = Response(status_code=200, headers=UPLOAD_RESPONSE_HEADERS)
|
||||
del response.headers["content-length"]
|
||||
return response
|
||||
else:
|
||||
try:
|
||||
from PIL import Image, ImageOps
|
||||
os.makedirs(thumb_dir, exist_ok=True)
|
||||
if (not os.path.exists(thumb_path)
|
||||
or os.path.getmtime(thumb_path) < os.path.getmtime(path)):
|
||||
im = Image.open(path)
|
||||
# iPhone / camera JPEGs encode rotation in EXIF rather than
|
||||
# the pixel data. Browsers honour that on the original via
|
||||
# image-orientation:from-image, but PIL strips EXIF when it
|
||||
# saves the JPEG thumb, leaving the pixels sideways. Bake
|
||||
# the rotation into the pixels before thumbnailing.
|
||||
im = ImageOps.exif_transpose(im)
|
||||
im.thumbnail((320, 320))
|
||||
if im.mode not in ("RGB", "L"):
|
||||
im = im.convert("RGB")
|
||||
im.save(thumb_path, "JPEG", quality=80)
|
||||
return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS)
|
||||
except Exception as e:
|
||||
logger.warning(f"Thumbnail generation failed for {file_id}: {e}")
|
||||
# Fall through to the full image.
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type=mime,
|
||||
|
||||
Reference in New Issue
Block a user