Merge current dev and refresh PR #6738 from the 7011 integration

2026-11-02 19:22:22 +01:00 · 2026-10-26 16:21:22 +00:00
parent 30430acfea 4d0610a1eb 39ca749f94
commit 444b60d14e
450 changed files with 60479 additions and 11016 deletions
@@ -11,7 +11,7 @@ from urllib.parse import urlparse
 from datetime import datetime
 from typing import Dict, Any, AsyncGenerator, List, Optional

-from fastapi import APIRouter, Request, HTTPException, Form, Query
+from fastapi import APIRouter, Request, HTTPException, Form, Query, Depends
 from fastapi.responses import StreamingResponse
 from pydantic import ValidationError

@@ -27,6 +27,7 @@ from src.llm_core import (
 from src.agent_loop import (
    stream_agent_loop,
    _configured_model_tool_surface,
+    _configured_model_tool_surface_raw,
    _local_media_needs_browser_render,
    _looks_like_workspace_coding_request,
 )
@@ -49,11 +50,17 @@ from src.foreground_model_routing import (
 from src.session_search import search_session_messages
 from src.prompt_security import untrusted_context_message
 from core.exceptions import SessionNotFoundError
-from src.auth_helpers import effective_user, get_current_user
+from src.auth_helpers import (
+    effective_user,
+    get_current_user,
+    is_delegated_credential,
+    require_api_token_scope,
+    require_chat_api_token_scope,
+)
 from routes.session_routes import _verify_session_owner
 from routes.document_helpers import _owner_session_filter
 from core.database import SessionLocal, get_session_mode, set_session_mode
-from core.database import Session as DBSession, ChatMessage as DBChatMessage
+from core.database import Session as DBSession, ChatMessage as DBChatMessage, Note
 from core.database import Document as DBDocument, ModelEndpoint
 from core.log_safety import redact_url
 from routes.research_routes import _resolve_research_endpoint
@@ -80,6 +87,8 @@ from src.tool_policy import (
    web_search_enabled_for_turn,
 )
 from src.tool_approvals import tool_approval_store
+from src.tool_approval_scopes import stamp_chat_session_grant
+from src.tool_security import delegated_credential_blocked_tools
 from src.workspace_paths import backend_workspace_path
 from src.client_tool_contract import TUI_CLIENT_TOOL_NAMES
 from src.model_profiles import (
@@ -320,13 +329,31 @@ def _is_external_discovery_request(text: str) -> bool:
    ))


+_URL_SCHEME_RE = re.compile(r"https?://", re.I)
+_PDF_URL_TAIL_RE = re.compile(r"\.pdf\b|/pdf/", re.I)
+
+
+def _mentions_pdf_url(value: str) -> bool:
+    """Same as ``re.search(r"https?://[^\\s]+(?:\\.pdf\\b|/pdf/)", value, re.I)``.
+
+    Checked per whitespace-free token from its first scheme only (a later
+    scheme's tail is a suffix of the first's), so a token packed with
+    `http://` repeats is scanned once instead of once per repeat (ReDoS).
+    """
+    for token in value.split():
+        scheme = _URL_SCHEME_RE.search(token)
+        if scheme and _PDF_URL_TAIL_RE.search(token, scheme.end() + 1):
+            return True
+    return False
+
+
 def _prefers_structured_document_tools(text: str) -> bool:
    """Identify external paper/PDF extraction where shell is a bad source route."""
    value = str(text or "")
    if re.search(r"(?:^|\s)(?:file://)?/workspace/[^\s`\"']+\.pdf\b", value, re.I):
        return False
    return bool(
-        re.search(r"https?://[^\s]+(?:\.pdf\b|/pdf/)", value, re.I)
+        _mentions_pdf_url(value)
        or re.search(
            r"\b(?:paper|report|study)\b[\s\S]{0,1200}?"
            r"\b(?:tables?|figures?|benchmarks?|scores?|metrics?)\b",
@@ -1265,6 +1292,23 @@ def _stream_failure_status(chunk: str) -> Optional[int]:
    return None


+def _reject_delegated_tool_approval(request: Request) -> None:
+    """Refuse an approval answered by a bearer API token.
+
+    A tool approval records that a HUMAN authorized one dangerous action. A
+    token is a delegated credential handed to an integration, so when it
+    answers the prompt it triggered, nobody is asked and the gate collapses
+    into an extra round trip. Owner and session already match here: the token
+    is answering on behalf of the account that minted it.
+    """
+    if is_delegated_credential(request):
+        raise HTTPException(
+            403,
+            "Tool approvals require an interactive session. "
+            "API tokens cannot authorize a gated action.",
+        )
+
+
 def _mark_tool_approval_resolved(sess, approval_id: Any, decision: Any) -> bool:
    """Persist a consumed approval decision on its existing tool event."""

@@ -1289,6 +1333,11 @@ def _mark_tool_approval_resolved(sess, approval_id: Any, decision: Any) -> bool:
            if str(ask_user.get("approval_id") or "") != approval_key:
                continue
            ask_user["resolved"] = normalized_decision
+            stamp_chat_session_grant(
+                ask_user,
+                getattr(sess, "id", ""),
+                normalized_decision,
+            )
            message_id = metadata.get("_db_id")
            resolved_metadata = {
                key: value for key, value in metadata.items() if key != "_db_id"
@@ -2106,6 +2155,68 @@ def _set_user_time_from_request(request: Request) -> None:
    except Exception:
        pass

+def _parse_selected_note_ids(raw) -> list[str]:
+    """Validate user-selected Note IDs before passing them to SQLAlchemy."""
+    if raw is None or raw == "":
+        return []
+    if not isinstance(raw, str) or len(raw) > 8192:
+        raise HTTPException(400, "Invalid selected_note_ids payload")
+    try:
+        parsed = json.loads(raw)
+    except (TypeError, ValueError) as exc:
+        raise HTTPException(400, "Invalid selected_note_ids JSON") from exc
+    if (
+        not isinstance(parsed, list)
+        or len(parsed) > 50
+        or any(
+            not isinstance(note_id, str)
+            or not note_id.strip()
+            or len(note_id) > 128
+            for note_id in parsed
+        )
+    ):
+        raise HTTPException(400, "selected_note_ids must be an array of up to 50 non-empty string IDs")
+    return list(dict.fromkeys(parsed))
+
+
+def _format_selected_notes_context(notes) -> str:
+    """Format selected Notes/Todos as reference context for the model."""
+    sections = []
+
+    for index, note in enumerate(notes, start=1):
+        lines = [
+            f"--- Selected Note {index} ---",
+            f"Title: {note.title or '(untitled)'}",
+            f"Type: {note.note_type or 'note'}",
+        ]
+
+        if note.content:
+            lines.extend([
+                "Content:",
+                str(note.content),
+            ])
+
+        if note.items:
+            try:
+                items = json.loads(note.items)
+            except (TypeError, json.JSONDecodeError):
+                items = []
+
+            if isinstance(items, list):
+                lines.append("Items:")
+                for item in items:
+                    if not isinstance(item, dict):
+                        continue
+
+                    status = "done" if item.get("done") else "pending"
+                    text = str(item.get("text") or "").strip()
+
+                    if text:
+                        lines.append(f"- [{status}] {text}")
+
+        sections.append("\n".join(lines))
+
+    return "\n\n".join(sections)

 def _resolve_prompt_thinking_mode(explicit_mode, preset_id, preset_manager):
    """Use an explicit request override, then fall back to the active preset."""
@@ -2131,13 +2242,17 @@ def setup_chat_routes(
    webhook_manager=None,
    skills_manager=None,
 ) -> APIRouter:
-    router = APIRouter(tags=["chat"])
+    router = APIRouter(
+        tags=["chat"],
+        dependencies=[Depends(require_chat_api_token_scope)],
+    )

    # ------------------------------------------------------------------ #
    # POST /api/chat (non-streaming)
    # ------------------------------------------------------------------ #
    @router.post("/api/chat", response_model=Dict[str, Any])
    async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str, Any]:
+        require_api_token_scope(request, "chat")
        _set_user_time_from_request(request)

        message = chat_request.message
@@ -2354,6 +2469,7 @@ def setup_chat_routes(
    # ------------------------------------------------------------------ #
    @router.post("/api/chat_stream")
    async def chat_stream(request: Request) -> StreamingResponse:
+        require_api_token_scope(request, "chat")
        body = None
        try:
            if request.headers.get("content-type", "").startswith("application/json"):
@@ -2371,6 +2487,8 @@ def setup_chat_routes(
        form_data = await request.form()
        message = form_data.get("message")
        session = form_data.get("session")
+        selected_note_ids = _parse_selected_note_ids(form_data.get("selected_note_ids"))
+
        attachments = form_data.get("attachments")
        use_web = form_data.get("use_web")
        use_research = form_data.get("use_research")
@@ -2473,7 +2591,7 @@ def setup_chat_routes(
            _explicit_web_intent = (
                _explicit_url_target
                or bool(re.search(
-                r"\b(search|look\s+(?:this|that|it|them|these|those)?\s*up|lookup|find\s*out|google|browse|web|online|latest|current|today|news|weather|forecast|rate|exchange\s+rate)\b",
+                r"\b(search|look\s+(?:(?:this|that|it|them|these|those)\s*)?up|lookup|find\s*out|google|browse|web|online|latest|current|today|news|weather|forecast|rate|exchange\s+rate)\b",
                _msg_l,
                ))
                or requires_external_web_verification(message)
@@ -2580,8 +2698,6 @@ def setup_chat_routes(
            or form_data.get("editor_session_id")
            or ""
        ).strip()
-        editor_state = form_data.get("editor_state") or ""
-        editor_screenshot = form_data.get("editor_screenshot") or ""
        photo_editor_requested = (
            str(form_data.get("photo_editor") or "").strip().lower()
            in {"1", "true", "yes", "on"}
@@ -2693,7 +2809,24 @@ def setup_chat_routes(
                current_rejected=workspace_rejected,
            )
            owner = effective_user(request)
+            selected_notes = []
+            if selected_note_ids:
+                notes_db = SessionLocal()
+                try:
+                    selected_notes = (
+                        notes_db.query(Note)
+                        .filter(
+                            Note.id.in_(selected_note_ids),
+                            Note.owner == owner,
+                        )
+                        .all()
+                    )
+                finally:
+                    notes_db.close()
+            selected_notes_context = _format_selected_notes_context(selected_notes)
+
            if tool_approval_id:
+                _reject_delegated_tool_approval(request)
                from src.agent_runtime.authority import require_user_approval_request
                require_user_approval_request(request)
                pending_tool_approval = tool_approval_store.peek(tool_approval_id)
@@ -2963,6 +3096,7 @@ def setup_chat_routes(
            sess, request, chat_handler, chat_processor,
            message=message,
            session_id=session,
+            selected_notes_context=selected_notes_context,
            preset_id=preset_id,
            att_ids=att_ids,
            use_web=use_web,
@@ -3163,10 +3297,6 @@ def setup_chat_routes(
            )
        )
        if _photo_editor_active:
-            # The editor tool family drives the browser-side image editor; it
-            # replaces the generic edit_image instruction. Without an open
-            # editor session the tools cannot reach the canvas, so tell the
-            # user to open the image instead of emitting failing tool calls.
            if editor_session:
                editor_context = {'role': 'system', 'content':
                    'The image editor is open (session ' + editor_session + '). Use run_editor_command, '
@@ -3236,6 +3366,12 @@ def setup_chat_routes(
        if _use_turn_contract and _turn_capabilities and "search_browser" not in _turn_capabilities:
            _explicit_web_intent = False
        disabled_tools = set()
+        # Minting is admin-only, so every owner-keyed check below answers
+        # "admin" for a token. Cap it at the non-admin policy instead.
+        # stream_agent_loop repeats this from delegated_credential.
+        _delegated_credential = is_delegated_credential(request)
+        if _delegated_credential:
+            disabled_tools.update(delegated_credential_blocked_tools())
        # Only disable bash when the caller *explicitly* set it to a falsy
        # value. When unset (None), defer to per-user privilege checks below.
        # Web search is per-turn opt-in: either the chat pre-search setting
@@ -3478,7 +3614,7 @@ def setup_chat_routes(
        # The strict router (spec) surface picks its tools in src/spec_router.py
        # AFTER authority is built, so its admission is keyed off the surface.
        try:
-            _spec_surface = _configured_model_tool_surface(
+            _spec_surface = _configured_model_tool_surface_raw(
                getattr(sess, "endpoint_url", ""), getattr(sess, "model", ""), _user
            ) == "spec"
        except Exception:
@@ -3490,15 +3626,8 @@ def setup_chat_routes(
            active_document=bool(active_doc),
            image_attachment=any(str(a.get('mime') or '').startswith('image/')
                                 for a in (ctx.preprocessed.attachment_meta or [])),
-            spec_surface=_spec_surface,
            capabilities=(({"photo_editor"} if _photo_editor_active else set()) | ({'search_browser'} if (
-                _explicit_browser_intent
-                or _external_discovery_intent
-                # Grant search/browser tools whenever the turn actually offers
-                # them (Web toggle on / not denied), not only for explicit
-                # browser wording. Otherwise the model sees web_search in its
-                # schema but every call is rejected by request authority.
-                or "web_search" not in disabled_tools
+                _explicit_browser_intent or _external_discovery_intent
            ) else set())),
        )
        if exact_tool_approval is not None:
@@ -3525,6 +3654,8 @@ def setup_chat_routes(
                                 if s["function"]["name"] != "host_shell"]
            _contract_mgr = get_mcp_manager()
            _owner_blocked = blocked_tools_for_owner(_user)
+            if _delegated_credential:
+                _owner_blocked.update(delegated_credential_blocked_tools())
            if (
                _workspace_tools_disabled_for_owner(_user)
                and not _native_workspace_contract
@@ -3723,17 +3854,10 @@ def setup_chat_routes(
                    )
            # Every recovery path receives the same scope denial. The central
            # dispatcher also checks the immutable contract after rewrites.
-            # The strict router (spec) surface picks its own tools AFTER this
-            # contract is resolved, so scoping disabled_tools to the regex
-            # contract would empty its menu (e.g. all email/notes tools denied
-            # for a plain "search for the appointment email" turn). On spec
-            # turns keep only the pre-contract denials (user/privilege/global/
-            # plan/compare) and let src/spec_router.py + the spec surface decide.
-            if not _spec_surface:
-                disabled_tools.update(
-                    s["function"]["name"] for s in _contract_schemas
-                    if not _turn_contract.permits(s["function"]["name"])
-                )
+            disabled_tools.update(
+                s["function"]["name"] for s in _contract_schemas
+                if not _turn_contract.permits(s["function"]["name"])
+            )
            # Contract resolution is the final policy-and-routing authority.
            # Some legacy/API-model paths arrive with a stale disabled snapshot
            # assembled before routing.  The scope-denial pass above may retain
@@ -4046,10 +4170,9 @@ def setup_chat_routes(
                        owner=_user,
                        size="auto",
                        progress_callback=_image_progress_callback,
-                        incognito=incognito,
                    ))
                else:
-                    _img_task = asyncio.create_task(do_generate_image(f"{_user_msg}\n{sess.model}\n512x512", session, owner=_user, incognito=incognito))
+                    _img_task = asyncio.create_task(do_generate_image(f"{_user_msg}\n{sess.model}\n512x512", session, owner=_user))
                _img_started = time.time()
                _img_tick = 0
                while not _img_task.done():
@@ -4161,8 +4284,6 @@ def setup_chat_routes(
                                    # indicator, but don't fold them into the saved
                                    # reply (mirrors the rewrite path below).
                                    if data.get("thinking"):
-                                        if thinking_mode == "off":
-                                            continue
                                        thinking_response += data["delta"]
                                    else:
                                        full_response += data["delta"]
@@ -4588,6 +4709,7 @@ def setup_chat_routes(
                        uploaded_files=ctx.uploaded_files,
                        defer_context_shaping=_foreground_policy.enabled,
                        external_untrusted_context_seen=external_untrusted_context_seen,
+                        delegated_credential=_delegated_credential,
                        exact_approval=exact_tool_approval,
                        client_runtime_context=client_runtime_context,
                        thinking_mode=thinking_mode,
@@ -4605,8 +4727,6 @@ def setup_chat_routes(
                                    # Forward them for the live indicator, but keep
                                    # them out of the saved reply (same as chat mode).
                                    if data.get("thinking"):
-                                        if thinking_mode == "off":
-                                            continue
                                        thinking_response += data["delta"]
                                    else:
                                        full_response = _render_state.content
@@ -4790,6 +4910,25 @@ def setup_chat_routes(
                                        _terminal_saved = True
                                        if _saved_id:
                                            yield f'data: {json.dumps(_render_state.message_saved(_saved_id))}\n\n'
+                                        run_post_response_tasks(
+                                            sess, session_manager, session, message, _saveable_no_tool_response,
+                                            _metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
+                                            incognito=incognito, compare_mode=compare_mode,
+                                            character_name=ctx.preset.character_name,
+                                            agent_rounds=_agent_rounds,
+                                            agent_tool_calls=_agent_tool_calls,
+                                            skills_manager=skills_manager,
+                                            owner=_user,
+                                            extract_skills=(
+                                                user_requested_agent
+                                                and not tool_approval_continuation
+                                            ),
+                                            allow_background_extraction=_post_response_extraction_allowed(
+                                                tools_blocked=tool_policy.block_all_tool_calls,
+                                                tool_approval_continuation=tool_approval_continuation,
+                                                client_runtime_context=client_runtime_context,
+                                            ),
+                                        )
                                    yield f'data: {json.dumps(_metrics_event)}\n\n'
                            except json.JSONDecodeError:
                                yield chunk
@@ -4898,20 +5037,13 @@ def setup_chat_routes(
        async def _safe_stream() -> AsyncGenerator[str, None]:
            """Wrapper that guarantees _active_streams cleanup even if stream_with_save
            raises before reaching a mode-specific finally block."""
-            _gen = stream_with_save()
            try:
-                while not _shutdown.is_shutting_down():
-                    try:
-                        chunk = await _gen.__anext__()
-                    except StopAsyncIteration:
+                async for chunk in stream_with_save():
+                    if _shutdown.is_shutting_down():
                        break
                    yield chunk
            finally:
                _active_streams.pop(session, None)
-                try:
-                    await _gen.aclose()
-                except Exception:
-                    pass

        # Compare panes and explicitly unattended native clients are
        # short-lived, single-shot generations with nobody to resume them.