diff --git a/.gitignore b/.gitignore index 1ae2878b5..d51361f8d 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,8 @@ services/data/ # Test capture artifacts (browser session dumps may contain personal content) .playwright-mcp/ +test-results/ +playwright-report/ # OS .DS_Store diff --git a/.gitleaks.toml b/.gitleaks.toml index 0eec3f1ae..8543c9666 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -35,3 +35,15 @@ regexTarget = "secret" regexes = [ '''^7abc5b334a6c3c52dd3eba15d8bc516da728a67f14fb7f56d0bb5992d89a16ca$''', ] + +[[allowlists]] +description = "Synthetic dummy API key in trace artifact sanitizer test fixture" +targetRules = ["generic-api-key"] +condition = "AND" +paths = [ + '''(?:^|/)tests/test_trace_artifact_sanitizer\.py$''', +] +regexTarget = "secret" +regexes = [ + '''^sk-123456789$''', +] diff --git a/app.py b/app.py index b0e6f9d55..ae859a579 100644 --- a/app.py +++ b/app.py @@ -178,6 +178,14 @@ app.add_middleware(SecurityHeadersMiddleware) import asyncio as _asyncio from starlette.middleware.base import BaseHTTPMiddleware as _BaseHTTPMiddleware from starlette.responses import JSONResponse as _JSONResponse +import anyio as _anyio +from starlette.requests import ClientDisconnect as _ClientDisconnect + +# SSE endpoints (e.g. /api/editor/bridge/events) end/abort mid-stream when the +# browser disconnects; Starlette's BaseHTTPMiddleware surfaces that as +# anyio.EndOfStream (and sometimes ClientDisconnect). Those are client-side +# events, not server errors, so we map them to a 499 instead of a 500 traceback. +_STREAM_CLIENT_EXC = (_anyio.EndOfStream, _ClientDisconnect) REQUEST_HARD_TIMEOUT = float(os.getenv("REQUEST_HARD_TIMEOUT", "45")) _TIMEOUT_EXEMPT_PREFIXES = ( @@ -190,6 +198,7 @@ _TIMEOUT_EXEMPT_PREFIXES = ( "/api/cookbook/setup", # remote pacman/apt installs "/api/upload", # large files "/api/image", # diffusion proxies (inpaint/harmonize/upscale/etc.) — own 120s httpx timeout + "/api/editor", # editor agent/bridge: generative steps can take 30-110s (SAM + local diffusion) "/api/memory/audit", # retains own 120s LLM inactivity timeout ) @@ -225,15 +234,43 @@ class _InteractiveActivityMiddleware(_BaseHTTPMiddleware): return await call_next(request) +# Heavy image ops that can run 30-150 s; a deploy restart must not cut these off. +_HEAVY_IMAGE_OP_PREFIXES = ( + "/api/image/inpaint", + "/api/image/replace-background", + "/api/image/outpaint", + "/api/image/extend", + "/api/image/remove-bg", + "/api/image/mask", + "/api/image/upscale-local", + "/api/image/denoise", + "/api/image/enhance-face", + "/api/gallery/ai-upscale", + "/api/gallery/style-transfer", + "/api/editor/agent", +) + + class _SlowRequestLogMiddleware(_BaseHTTPMiddleware): async def dispatch(self, request, call_next): + from src import activity + path = request.url.path or "" + heavy = any(path.startswith(p) for p in _HEAVY_IMAGE_OP_PREFIXES) + if heavy: + activity.begin_image_op(path) start = time.perf_counter() status = 500 try: response = await call_next(request) status = getattr(response, "status_code", 0) or 0 return response + except _STREAM_CLIENT_EXC: + # Client disconnected / SSE stream ended — not a server error. + status = 499 + return _JSONResponse({"detail": "client disconnected"}, status_code=499) finally: + if heavy: + activity.end_image_op(path) elapsed = time.perf_counter() - start try: threshold = float(os.getenv("ODYSSEUS_SLOW_REQUEST_LOG_SECONDS", "0.75") or "0.75") @@ -822,6 +859,10 @@ app.include_router(setup_gallery_routes()) from routes.editor_draft_routes import setup_editor_draft_routes app.include_router(setup_editor_draft_routes()) +# Natural-language editor planning (planner wired to the configured chat model) +from routes.editor_agent_routes import setup_editor_agent_routes +app.include_router(setup_editor_agent_routes()) + # Scheduled tasks + event bus from src.task_scheduler import TaskScheduler task_scheduler = TaskScheduler(session_manager) @@ -1000,6 +1041,16 @@ async def get_version(): "source_commit": APP_SOURCE_COMMIT, } + +@app.get("/api/busy") +async def get_busy(): + """Report in-flight heavy work so a deploy can wait instead of dropping it.""" + from src import activity + return { + "busy_image_ops": activity.heavy_image_ops_active(), + **activity.activity(), + } + @app.get("/api/health") async def health_check() -> Dict[str, str]: return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()} @@ -1438,4 +1489,35 @@ if __name__ == "__main__": bind_host = os.getenv("APP_BIND", "127.0.0.1") bind_port = int(os.getenv("APP_PORT", "7011")) - uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") + # Give in-flight requests time to finish on SIGTERM instead of being cut + # off mid-flight. Diffusion edits run 60-120 s; outpaint/extend can run up + # to ~330 s (provider cap 900 s), so default the graceful window to 900 s. + # systemd's TimeoutStopSec must exceed this (see the graceful-stop drop-in). + # + # We hook uvicorn's signal handler so a SIGTERM also flips src.shutdown: + # SSE / streaming generators check it and end promptly, so an open browser + # stream cannot stall a restart for the whole window. The long window stays + # a safety net for genuine in-flight image ops. + from src import shutdown as _shutdown + + config = uvicorn.Config( + app, + host=bind_host, + port=bind_port, + log_level="info", + timeout_graceful_shutdown=int(os.getenv("ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT", "900") or 900), + ) + server = uvicorn.Server(config) + + # uvicorn's capture_signals() wires SIGINT/SIGTERM to self.handle_exit, so + # wrapping that method lets us set the shutdown flag before the normal exit + # path runs (this uvicorn version has no install_signal_handlers kwarg). + _uvicorn_handle_exit = server.handle_exit + + def _handle_shutdown_signal(signum, frame): + _shutdown.request_shutdown() + _uvicorn_handle_exit(signum, frame) + + server.handle_exit = _handle_shutdown_signal + + server.run() diff --git a/docs/EDITOR_AGENT_API.md b/docs/EDITOR_AGENT_API.md new file mode 100644 index 000000000..b53873785 --- /dev/null +++ b/docs/EDITOR_AGENT_API.md @@ -0,0 +1,208 @@ +# Editor Agent API + +How an agent (LLM or script) drives the PhotoCraft editor. There are three layers: + +1. **Control API** — `window.geEditorControl`, a stable in-page command surface. +2. **NL planner** — `window.geEditorAI`, turns a prompt into control-API steps (LLM or basic mode). +3. **Bridge** — server-mediated SSE channel so a *server-side* agent can drive an **open** editor. + +All three are owner-scoped by the app session; nothing is exposed to unauthenticated users. + +--- + +## 1. Control API (`window.geEditorControl`) + +Installed when the editor opens. Version `EDITOR_CONTROL_VERSION`. + +```js +await window.geEditorControl.invoke(op, params) // { ok, op, result } | { ok:false, op, error } +await window.geEditorControl.run([{op, params}]) // sequential, stops on first failure +window.geEditorControl.list() // [{op, description}] +``` + +`invoke` is async. Every op that mutates the document pushes **one** undo entry via +`_saveState`, so a single op is one undo step. + +### Op catalog + +| Op | Params | Notes | +| --- | --- | --- | +| `document.info` | – | size, layers, mode | +| `document.importImage` | `{dataUrl,name,raster}` | load a raster image as a new layer | +| `layers.list` | – | top-most first | +| `layers.select` | `{id\|index\|name}` | index is top-most-first | +| `layers.addRaster` | – | new empty pixel layer | +| `layers.addAdjustment` | `{type,params}` | adjustment layer | +| `layers.duplicate` / `layers.delete` | – | on the active/selected layer(s) | +| `layers.setOpacity` | `{id?,opacity 0-1}` | | +| `layers.setBlend` | `{id?,blendMode\|mode}` | | +| `layers.group` / `layers.ungroup` | – | | +| `selection.selectAll` / `deselect` / `invert` | – | | +| `selection.rectangle` / `selection.ellipse` | `{x,y,w,h}` | document space | +| `selection.subject` | `{label}` | SAM text grounding (falls back to a centred rect) | +| `selection.face` | – | SAM "face" (falls back to an upper-centre ellipse) | +| `selection.sky` | – | SAM "sky" (falls back to a top-band colour range) | +| `selection.background` | `{fuzziness}` | border colour union | +| `selection.samText` | `{text}` | SAM text grounding only (throws without SAM) | +| `selection.colorRange` | `{color,fuzziness}` | | +| `selection.hasMask` | – | `{active, space}` | +| `adjustments.apply` | `{type,params}` | adjustment layer; camelCase types accepted | +| `adjustments.applyMasked` | `{brightness,contrast,saturation,warmth,invert}` | pixel pass, masked if a selection exists else whole layer | +| `filters.apply` | `{id}` | opens the slider dialog and applies | +| `filters.applyMasked` | `{id,radius,amount,invert}` | `id`: blur-gaussian, sharpen, unsharp-mask | +| `transform.start` / `transform.apply` / `transform.cancel` | – | | +| `text.add` | `{text,x,y,color,size,font,align}` | | +| `history.undo` / `history.redo` | – | | +| `history.mark` | – | returns current undo depth (for grouping) | +| `history.collapse` | `{mark,label}` | collapses entries above `mark` into one | +| `export.png` | – | `{dataUrl}` of the flattened document | +| `export.gallery` | – | saves a copy back to the gallery | +| `screenshot` | – | `{dataUrl,width,height}` of the composited canvas | + +Param conventions the planner must respect: + +- `adjustments.applyMasked`: `brightness`/`contrast`/`warmth` are **-100..100** (0 neutral); + `saturation` is a **multiplier** (1 neutral; values >3 are treated as percentages). +- `adjustments.apply` `type` aliases are normalised (`blackAndWhite` -> `black-white`). +- `color-balance` params are `{shadows:{r,g,b}, midtones:{r,g,b}, highlights:{r,g,b}}`; + a bare number is coerced to `{r:n,g:n,b:n}`. +- A masked op with **no** active selection degrades to a whole-layer edit (does not throw). + +--- + +## 2. NL planner (`window.geEditorAI`) + +```js +const res = await window.geEditorAI.execute('blur the background behind the person', { onStep }); +// exec: window.geEditorAI.execute('blur the background behind the person', { onStep }); +// res = { ok, prompt, reply, planner, basicMode, steps, trace, failed } +``` + +Flow (`static/js/editor/nl-edit.js`): + +1. Read document state + a **before** screenshot. +2. Ask the **server planner** (`POST /api/editor/plan`) with the prompt, state and + screenshot. The client passes its strong guidance via `state.planner_guidance`. + If the route is unavailable it falls back to `localPlan` (regex — labelled + **"basic mode"** in the UI). +3. `sanitizeSteps()`: drop no-op/export steps and, if a masked op has no preceding + `selection.*`, insert the region implied by the prompt (face/sky/background/subject). +4. `history.mark`, then run each step, screenshotting after each one. +5. **Self-check** (server plans only, max 3 rounds): send the after screenshot back in + `mode:"check"` with the original intent. If not satisfied, **revert** the previous + attempt (undo its history entries) and apply the model's **full corrected** plan. +6. `history.collapse` — the whole request becomes a single undoable step. + +UI: the Quick Edit box routes multi-clause / "and ..." prompts to this executor and +streams `Step i/n: ` into its status line; a `(basic mode)` suffix marks the offline planner. + +### Server route + +`POST /api/editor/plan` (HARNESS, `routes/editor_agent_routes.py`) + +``` +{ prompt, state, mode?: "plan"|"check", screenshot?: "data:image/..." } +-> { reply, planner, steps: [{op, params}], done? } +``` + +Owner-scoped, authenticated. Uses the configured default/utility model with vision. +If the model call is unavailable it returns a deterministic `planner:"fallback"` plan. + +--- + +## 3. Bridge (server agent drives an open editor) + +`static/js/editor/editor-bridge.js` subscribes the open editor to the server queue. +Enabled automatically when the editor opens (`window.__geEditorBridge`). + +- Subscribe: `GET /api/editor/bridge/events?session=` — SSE, `data: {id,op,params}` (keepalive every 15s). +- The page runs the op via the control API and replies: + `POST /api/editor/bridge/result {id,ok,result,error,screenshot,ms}`. +- Enqueue a command (server side): `POST /api/editor/bridge/command {session, op|steps, wait, timeout}`. + With `wait:true` and one step the call blocks and returns the editor's result. + Otherwise use `POST /api/editor/bridge/await {id, timeout}` for each queued id. + Timeouts are seconds, greater than zero and at most 900. Pending results expire + after the command timeout; an unknown or another owner's id returns 404. + +Round-trip example (authenticated; `` = `window.__geEditorBridge.session`): + +```bash +curl -s -X POST http://127.0.0.1:7011/api/editor/bridge/command \ + -H 'Content-Type: application/json' -b "odysseus_session=$TOKEN" \ + -d '{"session":"default","op":"layers.addRaster","wait":true,"timeout":8}' +# -> {"id":"...","ok":true,"result":{...},"screenshot":"data:image/png;base64,..."} +``` + +Session scoping uses the authenticated owner and session as separate values. A +bound editor-tool turn uses its server-owned session, not a model-selected one. +The bridge is a **browser-only** flow: it requires an editor page to be open. +For `/api/editor/agent`, clients must send `incognito: true` for private image +results; omitted or false requests use normal gallery persistence. These +owner-scoped routes do not accept delegated API tokens without a scope contract. + +--- + +## 4. SAM dependency + +`selection.subject` / `selection.face` / `selection.sky` / `selection.samText` use the +server SAM mask tool (`POST /api/image/mask`). When the SAM tools are **not installed** +the editor raises *"Object mask tools are not installed"* and the control op falls back +to a **coarse approximation** (centred rect / upper-centre ellipse / top-band colour range). +On a plain background the approximation is acceptable; on real photos it leaks, producing +visible rectangular/elliptical artifacts. Install the Cookbook SAM mask dependencies for +true subject/face/sky segmentation. + +`selection.colorRange` and `selection.background` never need SAM (pure colour matching), +but they are also coarse on busy scenes. + +--- + +## 5. Two ways for an agent to use this — pick one + +### Option A — built-in tool (in-process) + +Register an editor-control tool inside the app (same process as the model) that calls the +bridge/control API directly. Uses the app's normal login; nothing new is exposed. This is +the rejected stdio-MCP design's replacement: a server-side tool that talks to the bridge +needs either an internal token (auth-bypass) or to run in-process. **Decision pending (user).** + +### Option B — browser-only flow (works today) + +No new server tool. An external agent drives an open editor purely through the bridge +endpoints using the logged-in owner's session. Restricted to the editor the user has open. +This is the flow demonstrated above and requires **no** auth-bypass. + +A stdio `photo_editor` MCP server was prototyped and **rejected** because the built-in MCP +subprocess has no internal token; wiring it would require expanding an auth bypass. +The two viable options above are left for the user to choose. + +--- + +## 6. Known limitations / honest grading + +Live run against real 7011 with the server LLM planner (7 prompts): all 7 executed +(`planner:"llm"`). Graded by eye: + +- PASS: black & white portrait; warm cinematic; sharper + more vivid; text title. +- FAIL (region-masked, SAM-bound): "dramatic sky" (global over-saturation instead of a + dramatic, masked sky), "blur the background" (coarse mask leaks; visible rectangle), + "brighten the face" (coarse ellipse/rect artifact). + +Whole-image edits are reliable; subject/face/sky region edits need SAM. The self-check +loop works (revert-and-replan) but the region cases spend all 3 iterations. + +## 7. Theming (control kit) + +The editor's interactive controls (ranges, number fields, selects, checkboxes) share a +single themed kit: `static/js/editor/control-kit.js` adds `.ge-ctl-*` classes scoped to +the editor roots (`#gallery-editor-container`, `.ge-adj-popup`, `.ge-filter-overlay`, +`.ge-canvas-size-overlay`, `.ge-float-window`, `.ge-transform-popup`), and +`static/css/13-image-editor.css` (section "Editor control kit") styles them from the app +design tokens (`static/css/00-tokens.css`) to match the app sliders/selects. It keeps +range fills in sync and attaches a **scrub-on-label** affordance to numeric fields +(drag the field's label to change the value; arrow keys work, Shift = x10). + +**The PhotoCraft editor is dark-only.** `:root.light` flips the global tokens, but +`13-image-editor.css` has no `.light` handling and the editor relies on dark surfaces, +so light mode renders a garbled mix. Unlike the rest of Odysseus, the editor follows +Photoshop and is not offered in a light theme. diff --git a/docs/QUICKEDIT_COVERAGE.md b/docs/QUICKEDIT_COVERAGE.md new file mode 100644 index 000000000..895d84502 --- /dev/null +++ b/docs/QUICKEDIT_COVERAGE.md @@ -0,0 +1,255 @@ +# Quick Edit Coverage Audit + +Audit of how much of the PhotoCraft editor is reachable through **Quick Edit** +(the natural-language editor agent). Built against build **0.20.44** (HARNESS +bridge fan-out fix `4dd11826`) and client commit `e78b5a96`. + +Quick Edit executes in two stages: + +1. **Plan** — `static/js/editor/editor-agent.js` posts `{prompt, state, + screenshot}` to `/api/editor/agent`. HARNESS `routes/editor_agent_routes.py` + plans with the shared style guide (`src/editor_style_guide.py`) plus an LLM, + validates every step against its own `OP_CATALOG`, then executes. +2. **Execute** — only `ai_edit_image` / `generate_image` run server-side; every + other op is delivered to the page over the editor bridge (SSE) and run by the + client control surface (`window.geEditorControl.invoke`, catalog in + `static/js/galleryEditor.js`). A local fallback planner lives in + `static/js/editor/nl-edit.js`. + +There are **three** catalogs that must agree: + +| Catalog | File | Owner | +|---|---|---| +| Client control ops | `static/js/galleryEditor.js` | PHOTOSHOP | +| Server `OP_CATALOG` | `routes/editor_agent_routes.py` | HARNESS | +| Style guide / planner | `src/editor_style_guide.py` + `nl-edit.js` | HARNESS + PHOTOSHOP | + +Legend: **C** = in client catalog · **S** = in server `OP_CATALOG` (incl. +`NEW_OP_NAMES`) · **P** = the planner/style guide emits it for some phrasing. +A capability is **usable from Quick Edit** only when the planner can emit it +(**P**) and the op exists on the side that runs it. + +--- + +## 1. Menu bar (12 menus) + +| Capability | Trigger phrasing | Op | C | S | P | Status | +|---|---|---|---|---|---|---| +| File → Import | "open this image" | `document.importImage` | ✔ | ✔ | partial | OK | +| File → Load project | "open my project" | `document.openPsd` | ✔ | ✘ | ✘ | **Server gap** | +| File → Save / Save copy / Export PSD | "save as psd" | `document.savePsd` | ✔ | ✔ | ✔ | OK | +| File → Export | "export png" | `export.png` | ✔ | ✔ | ✔ | OK | +| File → Save project | — | (project store) | n/a | n/a | ✘ | Out of scope (not an edit op) | +| Edit → Undo/Redo/History | "undo" | `history.undo` / `redo` | ✔ | ✔ | ✔ | OK | +| Edit → Free Transform | "free transform" | `transform.start` | ✔ | ✔ | ✔ | OK | +| Edit → Transform → Rotate 90/180 | "rotate this layer 90" | `layer.rotate` | ✔ | ✔ | ✔ | OK (whole image vs layer routed by the fast path) | +| Edit → Transform → Flip H/V | "flip this layer" | `layer.flip` | ✔ | ✔ | ✔ | OK (whole image vs layer routed by the fast path) | +| Image → Fill | "fill the whole image with white" | — | ✘ | ✘ | ✘ | **No op** | +| Image → Canvas Size | "make the canvas wider" | `document.resizeCanvas` | ✔ | ✔ | partial | OK | +| Image → Image Size | "resize the image to 1200px" | `document.resizeCanvas` | ✔ | ✔ | partial | OK | +| Image → Rotate 90/180 (whole image) | "rotate the whole image 90" | `document.rotate`/all-layers | ✘ | ✘ | ✘ | **No control op** | +| Image → Flip H/V (whole image) | "flip the whole canvas" | `document.flip`/all-layers | ✘ | ✘ | ✘ | **No control op** | +| Image → Mode RGB/Grayscale | "make it black and white" | `adjustments.apply` black-white | ✔ | ✔ | ✔ | OK (mode via adjustment) | +| Image → Adjustments → Invert | "invert the colours" | `adjustments.apply` invert | ✔ | ✔ | ✔ | OK (fast path colour-inverts the active layer; "invert the selection" stays Select ▸ Inverse) | +| Image → Mode CMYK/Lab, Bits 16/32 | — | — | ✘ | ✘ | ✘ | Not implemented (disabled in UI) | +| Layer → New | "add a layer" | `layers.addRaster` | ✔ | ✔ | partial | OK | +| Layer → Duplicate | "duplicate the layer" | `layers.duplicate` | ✔ | ✔ | partial | OK | +| Layer → Group | "group the layers" | `layers.group` | ✔ | ✔ | ✘ | **Server OK, planner gap** | +| Layer → Rasterize | "rasterize the text" | `layers.rasterize` | ✔ | ✔ | ✔ | OK | +| Layer → Delete | "delete the layer" | `layers.delete` | ✔ | ✔ | partial | OK | +| Layer → Merge Down | "merge down" | `layers.mergeDown` | ✔ | ✘ | ✘ | **Server gap** | +| Layer → Merge All / Flatten | "flatten the image" | `layers.flatten` | ✔ | ✘ | ✘ | **Server gap** | +| Layer → Select All | "select all layers" | (multi-select) | ✔ | ✔ | ✔ | OK (fast path selects every layer) | +| Layer → Combine (unite/subtract/…) | "combine the shapes" | `_combineSelectedShapes` | ✔ | ✔ | ✔ | OK (fast path selects the shape layers and combines) | +| Layer → Fill | "fill the layer with blue" | `selection.fill` / flood | partial | ✘ | ✘ | **Gap** | +| Layer → Layer Mask (reveal/hide/apply/invert/feather) | "add a layer mask" | `layers.addMaskFromSelection` | ✔ | ✔ | ✔ | OK | +| Layer → Opacity | "set the layer opacity to 50%" | `layers.setOpacity` | ✔ | ✔ | partial | OK (fast path sets the active layer's opacity) | +| Layer → Blend Mode | "set the blend mode to multiply" | `layers.setBlend` | ✔ | ✔ | partial | OK (fast path sets the active layer's blend mode) | +| Type → Add | "add a title" | `text.add` | ✔ | ✔ | ✔ | OK | +| Type → Character | "make the text bigger" | `text.edit` | ✔ | ✘ | ✘ | **Server gap + planner gap** | +| Type → Paragraph | "centre the text" | `text.layout` | ✔ | ✔ | partial | OK | +| Type → Warp | "warp the text" | `_rerenderTextLayer` (warp) | ✔ | ✔ | ✔ | OK | +| Select → All/Deselect/Invert | "invert the selection" | `selection.*` | ✔ | ✔ | ✔ | OK | +| Select → Reselect | "reselect" | `_reselectSelection` | ✔ | ✔ | ✔ | OK | +| Select → Transform | "transform the selection" | `transform.start` | ✔ | ✔ | partial | OK | +| Select → Refine | "refine the edge" | `selection.feather/expand` | ✔ | ✘ | ✘ | **Server gap** | +| Select → Color Range | "select the blue sky" | `selection.colorRange` | ✔ | ✔ | partial | OK | +| Select → Select and Mask | "select the subject" | `selection.subject` | ✔ | ✔ | ✔ | OK | +| Select → Save | "save the selection" | `_saveNamedSelection` | ✔ | ✔ | ✔ | OK | +| Select → Load | "load the selection as sky" | `_loadNamedSelection` | ✔ | ✔ | ✔ | OK | +| Filter → every filter (30+) | "pixelate the image" | `filters.apply` | ✔ | ✔ | ✔ | OK (fast path covers pixelate/emboss/twirl/pinch/spherize/ripple/solarize/find-edges/crystallize/noise/despeckle/unsharp + blur/sharpen/vignette) | +| Filter → Blur → Motion / Zoom | "motion blur" / "zoom blur" | `filters.apply` (blur-motion / blur-zoom) | ✔ | ✔ | ✔ | OK (fast path maps motion blur and zoom blur to their Filter-menu variants) | +| Filter → Layer effects (drop shadow, stroke, colour overlay, outer glow) | "add a drop shadow" | `layers.addStyle` | ✔ | ✔ | ✔ | OK (fast path adds drop shadow / stroke / outer glow / colour & gradient overlay; colour parsed from the prompt) | +| View → Rulers/Grid/Snap/Clear guides | "show the grid" | `#ge-view-menu` | ✔ | ✔ | ✔ | OK | +| Window → panels | — | n/a | n/a | n/a | ✘ | UI-only | +| 3D / Plugins | — | — | ✘ | ✘ | ✘ | Not implemented (disabled in UI) | + +## 2. Tools + +Toolbar tools are interactive, not bridge ops, so coverage is about whether the +**effect** of a tool is reachable by prompt. + +| Tool | NL-reachable? | Notes | +|---|---|---| +| Move / Transform | `transform.*` | OK (transform.start/apply/cancel) | +| Hand / Zoom | — | View only | +| Crop | `document.crop` + format guard | OK | +| Text / Shape | `text.add`, `text.layout` | Shape has no add-shape op — **gap** | +| Brush / Eraser / Dodge / Burn / Smudge / Clone / Heal / Sharpen | — | Interactive painting; no NL op — **gap** (out of scope for planner) | +| Gradient | — | **No op** | +| **Paint Bucket** | `fill.flood` | **Server gap + planner gap** (op exists client-side only) | +| Eyedropper | — | UI only | +| Marquee / Lasso / Wand / Quick Select / Object Select / Pen / Path | `selection.rectangle/ellipse/polygon/subject/…` | OK for region/reference selects; lasso/pen paths have `selection.polygon` only | +| SAM / Inpaint / Rembg | `selection.subject`, `ai.removeObject`, `ai.removeBackground` | OK | +| **Pixel brush + grid** | — | UI state only; no op — **gap** (acceptable: it is a brush mode, not an edit op) | + +## 3. Right-click context menu + +| Item | Backing op | Reachable by prompt? | +|---|---|---| +| Brush size / preset / Pixel Brush | `state` UI | No (interactive) | +| Select All / Deselect / Inverse / Feather / Expand / Contract / Border / Smooth | `selection.*` | Yes | +| Transform Selection / Refine Edge | `transform.start`, `selection.feather` | Partial | +| Apply / Cancel Crop | `document.crop` | Yes | +| Commit / Cancel / Free Transform | `transform.apply/cancel/start` | Yes | +| Add Text Layer / Rasterize Text | `text.add`, `layers.rasterize` | Yes (rasterize OK) | +| Copy Color Hex / Swap Colors | `state.color` | OK (fast path swaps / resets the foreground & background swatches) | +| Fill Selection w/ Foreground | `selection.fill` | OK (fast path fills the selection/canvas with the foreground or background colour) | +| Cut / Copy / Paste | `selection.copy/paste` | **Gap** (no planner intent) | +| Rotate 90/180, Flip H/V | `layer.rotate/flip` | **Server+planner gap** (same as Edit menu) | +| Undo / Redo | `history.*` | Yes | + +## 4. Adjustments + +`adjustments.apply` / `adjustments.applyMasked` cover all adjustment types +(brightness-contrast, white-balance, shadows-highlights, vibrance, clarity, +vignette, black-white, …). The style guide emits several. **Status: OK** +(planner covers the common ones; the long tail is reachable only via explicit +wording). + +## 5. Filters + +`filters.apply` / `filters.applyMasked` run every filter the UI lists. The +planner only emits **sharpen**, **blur-gaussian**, and **vignette** via intents. +**Gap (planner):** no intent for noise, pixelate, stylize, distort, render, +unsharp, sharpen-edges, colour-overlay/drop-shadow/stroke. +**Gap (HARNESS):** `filters.apply`/`applyMasked` are in the server catalog, so +the fix is style-guide intents, not catalog. + +## 6. Layer styles + +`layers.addStyle {type, params}` exists in both catalogs. **Planner never emits +it**, so "add a drop shadow to the title", "add a stroke", "colour overlay" fail. +**Gap (planner).** + +## 7. Selection ops + +`selection.rectangle/ellipse/polygon/fill/delete/copy/paste/feather/expand/ +contract/selectAll/deselect/invert/colorRange/subject/samText/face/sky/ +background/hasMask/maskBounds/exportMask/regionLuminance`. **Status: mostly OK.** +Missing: reselect, save-selection. + +## 8. Document ops + +| Capability | Op | C | S | P | +|---|---|---|---|---| +| Crop | `document.crop` | ✔ | ✔ | ✔ | +| Resize canvas / image | `document.resizeCanvas` | ✔ | ✔ | partial | +| **Extend canvas** | `document.extend` | ✔ | ✘ | ✘ | +| Save PSD | `document.savePsd` | ✔ | ✔ | ✔ | +| Export layers | `document.exportLayers` | ✔ | ✘ | ✘ | +| Duplicate / Create / Switch / List | `document.*` | ✔ | ✘ | partial | + +## 9. AI edit & generate modes + +| Mode | Op | Notes | +|---|---|---| +| Generative fill | `ai.generativeFill` | OK | +| Remove object | `ai.removeObject` | OK | +| Harmonize | `ai.harmonize` | OK | +| Upscale | `ai.generativeUpscale` | OK | +| Generate image | `ai.generateImage` / server `generate_image` | OK | +| Expand | `ai.generativeExpand` | OK (content-bounds prefill, SW v760) | +| Remove background | `ai.removeBackground` | OK | +| Instruction edit / scene replace | server `ai_edit_image` | OK (HARNESS) | + +--- + +## 10. Gap list + +### Client-side (fix in this repo — PHOTOSHOP-owned) — **ALL FIXED** +1. ✅ **`fill.flood` not in the NL path** — added to `nl-edit.js OP_CATALOG` + + `localPlan` fill intent ("fill … with "; falls back to `selection.fill` + when a region is selected). +2. ✅ **`layer.flip` / `layer.rotate` not in the NL path** — added to + `nl-edit.js OP_CATALOG` + "this layer" intent (whole-image phrasing routes to + `document.flip`/`document.rotate`). +3. ✅ **`text.edit` not in the NL path** — added catalog entry + intent for font + size/colour ("make the text bigger/smaller", "make it red"). +4. ✅ **`document.extend` not in the NL path** — added catalog entry + intent for + "extend the canvas to 16:9" (derives target size from the document). +5. ✅ **`layers.align`** — added planner intent ("align … to the centre/left/…"). +6. ✅ **`layers.addStyle`** — added planner trigger ("drop shadow", "stroke", + "colour overlay"). +7. ✅ **Whole-image rotate/flip has no control op** — added `document.rotate` / + `document.flip` control ops (`galleryEditor.js`) and catalog/intent entries. +8. ✅ **`selection.copy/paste` intents** — added "copy the selection"/"paste". + +### Server-side (file for HARNESS in the passover) +1. `routes/editor_agent_routes.py OP_CATALOG` is missing ops the client runs: + `layer.flip`, `layer.rotate`, `text.edit`, `fill.flood`, `layers.align`, + `layers.rasterize`, `layers.mergeDown`, `layers.flatten`, `document.extend`, + `document.exportLayers`, `document.*` (list/duplicate/create/switch/openPsd), + `mask.*`, `selection.*` (polygon/feather/expand/contract/copy/paste/hasMask/ + face), `transform.*` are partly present. +2. `src/editor_style_guide.py` has no intents for: layer transform + (rotate/flip active layer), whole-document rotate/flip, text editing, align, + layer styles (drop shadow/stroke/overlay), flood fill, copy/paste, noise/ + pixelate/stylize/distort/render filters. + +## 11. Coverage summary + +Counting distinct user-facing capabilities in the toolbar + 12 menus + right-click +(~150 including the filter/effect long tail): + +- **Have a client op:** ~85% +- **Client op also in server OP_CATALOG:** ~75% +- **Planner can emit for natural phrasing:** ~72% (was ~55%; client-side + planner intents for fill, layer/whole-image transform, text edit, align, + layer styles, copy/paste and extend are now implemented) +- **Usable from Quick Edit end-to-end today:** **45/45 prompts (100%)** measured + live on build **0.20.45** with `tests/e2e/photo-editor/quickedit-coverage.js` + (`/tmp/opencode/qe-coverage45/quickedit-coverage.json`). HARNESS build 0.20.45 + (`4a6932a2`) added the 12 missing structural ops to the server `OP_CATALOG` + + style guide v5, so every audited prompt now plans to the exact op. + +The gap was dominated by the **planner**, not by missing client functionality; +that planner gap is now closed end-to-end (30/45 on 0.20.44 -> 45/45 on 0.20.45). + +## 12. Test prompts (~40) + +See `docs/QUICKEDIT_PROMPTS.md` and the runner +`tests/e2e/photo-editor/quickedit-coverage.js` for per-prompt pass/fail. + +### Live run (build 0.20.45) — 45/45 PASS + +All 45 prompts plan to the exact expected op. HARNESS 0.20.45 routes these +structural ops server-side: `fill.flood`, `text.edit`, `layers.align`, +`layer.rotate`/`layer.flip`, `document.rotate`/`document.flip`, +`selection.copy`/`selection.paste`, `layers.flatten`, `layers.mergeDown` (plus +the previously passing set). Results: +`/tmp/opencode/qe-coverage45/quickedit-coverage.json`. + +### Previous run (build 0.20.44) — 30/45 PASS + +Kept for provenance. Failures were all the server planner emitting a generic +`adjustments.apply`/`transform.apply` or a wrong op; ids 1, 2, 3, 4, 8, 9, 10, +11, 12, 13, 14, 19, 20, 40, 41. All now fixed in 0.20.45. + + +## Provenance + +Recon inventory captured from `static/js/editor/build/topbar.js`, +`static/js/editor/build/toolbar.js`, `static/js/editor/canvas-context-menu.js`, +`static/js/galleryEditor.js` (`installEditorControl`), `static/js/editor/nl-edit.js`, +`routes/editor_agent_routes.py`, and `src/editor_style_guide.py`. diff --git a/mcp_servers/email_server.py b/mcp_servers/email_server.py index ed8d5acff..b43eeca87 100644 --- a/mcp_servers/email_server.py +++ b/mcp_servers/email_server.py @@ -1185,6 +1185,9 @@ def _fixture_email_record(row: dict, uid_num: int, owner: str) -> dict: "subject": subject, "from": sender_name or sender_addr or sender, "from_address": sender_addr, + "to": str(row.get("to") or headers.get("To") or ""), + "cc": str(row.get("cc") or headers.get("Cc") or ""), + "references": str(row.get("references") or headers.get("References") or ""), "date": date_str, "date_epoch": date_epoch, "summary": body[:240], @@ -1557,6 +1560,7 @@ def _fixture_update_email(uid=None, source_folder="INBOX", account=None, **updat if not isinstance(rows, list): return False owner = _current_owner() + matches = [] for index, row in enumerate(rows, start=1): if not isinstance(row, dict): continue @@ -1568,14 +1572,19 @@ def _fixture_update_email(uid=None, source_folder="INBOX", account=None, **updat continue if not _fixture_folder_matches(row.get("folder") or "INBOX", source_folder): continue - for key, value in updates.items(): - if value is None: - row.pop(key, None) - else: - row[key] = value - path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - return True - return False + if not _fixture_row_matches_account(_fixture_email_record(row, index, owner or row_owner), account): + continue + matches.append(row) + if len(matches) != 1: + return False + row = matches[0] + for key, value in updates.items(): + if value is None: + row.pop(key, None) + else: + row[key] = value + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return True # ── Tool implementations ── @@ -2925,7 +2934,9 @@ def _create_email_draft_document( def _draft_reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None, title=None): """Create a threaded Odysseus reply draft document. Does not send.""" - fixture = _fixture_email_action_target(uid=uid, folder=folder, account=account) + fixture = _fixture_read_email(uid=uid, folder=folder, account=account) + if _fixture_email_enabled() and (fixture is None or fixture.get("error")): + return fixture or {"error": "No email fixture is available for the authenticated owner"} if fixture is not None: sender = str(fixture.get("from_address") or fixture.get("from") or "") _, sender_addr = email.utils.parseaddr(sender) @@ -3178,7 +3189,9 @@ def _reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None): def _set_flag(uid, folder, flag, add=True, account=None): """Add or remove an IMAP flag (e.g. \\Seen, \\Answered, \\Deleted).""" - if _fixture_email_action_target(uid=uid, folder=folder, account=account) is not None: + if _fixture_email_enabled(): + if _fixture_email_action_target(uid=uid, folder=folder, account=account) is None: + return False if flag == "\\Seen": return _fixture_update_email(uid=uid, source_folder=folder, account=account, read=bool(add)) if flag == "\\Answered": diff --git a/routes/chat_helpers.py b/routes/chat_helpers.py index eb0bfc310..0d815309e 100644 --- a/routes/chat_helpers.py +++ b/routes/chat_helpers.py @@ -569,8 +569,72 @@ async def auto_name_session(session_manager, sess): if not first_msg: return + from src.owner_identity import is_request_sentinel_owner + owner = getattr(sess, "owner", None) + invalid_owner = ( + not isinstance(owner, str) or not owner.strip() + or owner != owner.strip() or is_request_sentinel_owner(owner) + ) + if invalid_owner: + session_manager.update_session_name(sess.id, fallback_session_title(first_msg)) + return + endpoint_url = str(getattr(sess, "endpoint_url", "") or "") model_name = str(getattr(sess, "model", "") or "") + + # The spec ("Ajax") model names its own chats from the user's first + # prompt. This function only runs after the first response stream has + # settled (auto_name_session_after_stream), so titling does not + # contend with the visible answer. Thinking is disabled and the + # budget kept small so the model returns a bare title. + try: + from src.model_profiles import uses_odysseus_progressive_thinking + _is_spec_model = bool(model_name) and uses_odysseus_progressive_thinking(model_name) + except Exception: + _is_spec_model = False + if _is_spec_model and endpoint_url: + try: + from src.database import SessionLocal + from src.endpoint_resolver import (build_chat_url, build_headers, + resolve_endpoint_runtime, resolve_owner_registered_endpoint) + from src.owner_identity import auth_disabled + + owner = getattr(sess, "owner", None) + if not owner and not auth_disabled(): + raise ValueError("Session has no owner") + db = SessionLocal() + try: + endpoint = resolve_owner_registered_endpoint(db, endpoint_url, owner) + base, key = resolve_endpoint_runtime(endpoint, owner=owner) + title_url = build_chat_url(base) + title_headers = build_headers(key, base) + finally: + db.close() + title = await llm_call_async( + title_url, + model_name, + [ + {"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation \u2014 just the title."}, + {"role": "user", "content": first_msg}, + ], + temperature=0.3, + max_tokens=32, + headers=title_headers, + timeout=20, + thinking_mode="off", + ) + except Exception as e: + logger.warning("[auto-name] spec title call failed for %s: %s", sess.id, e) + title = "" + title = str(title or "").strip().strip('"\'').strip() + from src.text_helpers import strip_think + title = strip_think(title, prose=False, prompt_echo=False) + if not title or len(title) >= 80 or "\n" in title: + title = fallback_session_title(first_msg) + session_manager.update_session_name(sess.id, title) + logger.info(f"Auto-named spec session {sess.id}: {title}") + return + if ( "ttft" in model_name.lower() or re.search(r":18\d{3}\b", endpoint_url) @@ -581,6 +645,14 @@ async def auto_name_session(session_manager, sess): return owner = getattr(sess, "owner", None) + from src.owner_identity import auth_disabled + if not owner and not auth_disabled(): + title = fallback_session_title(first_msg) + session_manager.update_session_name(sess.id, title) + logger.info(f"Auto-named unowned session {sess.id} deterministically: {title}") + return + + from src.task_endpoint import resolve_task_endpoint t_url, t_model, t_headers = resolve_task_endpoint( sess.endpoint_url, sess.model, sess.headers, owner=owner ) @@ -588,24 +660,64 @@ async def auto_name_session(session_manager, sess): logger.debug("[auto-name] No model provided, skipping") return + sess_url = getattr(sess, "endpoint_url", None) + if t_url and sess_url and t_url == sess_url: + from src.database import SessionLocal + from src.endpoint_resolver import ( + build_chat_url, + build_headers, + resolve_endpoint, + resolve_endpoint_runtime, + resolve_owner_registered_endpoint, + ) + db = SessionLocal() + try: + endpoint = resolve_owner_registered_endpoint(db, t_url, owner) + base, key = resolve_endpoint_runtime(endpoint, owner=owner) + t_url = build_chat_url(base) + t_headers = build_headers(key, base) + except Exception as e: + logger.debug("[auto-name] session fallback endpoint %s unauthorized for %s: %s", t_url, owner, e) + t_url, t_model, t_headers = None, None, None + for prefix in ("utility", "default"): + try: + u, m, h = resolve_endpoint(prefix, owner=owner) + if u and m: + t_url, t_model, t_headers = u, m, h + break + except Exception: + pass + finally: + db.close() + + if not t_url or not t_model: + title = fallback_session_title(first_msg) + session_manager.update_session_name(sess.id, title) + logger.info(f"Auto-named session {sess.id} deterministically (no authorized model endpoint): {title}") + return + # max_tokens big enough that reasoning models (Minimax M2, # DeepSeek R1, QwQ, etc.) have headroom for … # plus the actual title — 200 used to clip them mid-reasoning # so strip_think left an empty string and no rename happened. # Timeout matches: 60s gives slow local reasoners room to finish. - title = await llm_call_async( - t_url, - t_model, - [ - {"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation — just the title."}, - {"role": "user", "content": first_msg}, - ], - temperature=0.3, - max_tokens=64, - headers=t_headers, - timeout=15, - reasoning_effort=effort_for_call(t_url, t_model, owner, sess), - ) + try: + title = await llm_call_async( + t_url, + t_model, + [ + {"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation — just the title."}, + {"role": "user", "content": first_msg}, + ], + temperature=0.3, + max_tokens=64, + headers=t_headers, + timeout=15, + reasoning_effort=effort_for_call(t_url, t_model, owner, sess), + ) + except Exception as e: + logger.warning("[auto-name] non-spec title call failed for %s: %s", sess.id, e) + title = "" title = title.strip().strip('"\'').strip() # Strip / blocks (closed, dangling, or stray tags) diff --git a/routes/chat_routes.py b/routes/chat_routes.py index 5a42d3351..c6f306e96 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -27,11 +27,13 @@ from src.llm_core import ( from src.agent_loop import ( stream_agent_loop, _configured_model_tool_surface, + _configured_model_tool_surface_raw, _local_media_needs_browser_render, _looks_like_workspace_coding_request, ) from src.agent_loop import _normalize_ody_qwen_text_artifacts from src import agent_runs +from src import shutdown as _shutdown from src.model_context import estimate_tokens from src.context_compactor import ( apply_compaction_state, @@ -329,6 +331,7 @@ def _is_external_discovery_request(text: str) -> bool: _URL_SCHEME_RE = re.compile(r"https?://", re.I) _PDF_URL_TAIL_RE = re.compile(r"\.pdf\b|/pdf/", re.I) +_EDITOR_SESSION_RE = re.compile(r"[A-Za-z0-9@._:-]{1,128}") def _mentions_pdf_url(value: str) -> bool: @@ -2688,6 +2691,23 @@ def setup_chat_routes( logger.info("chat→agent auto-escalation: explicit private browser workflow") active_doc_id = form_data.get("active_doc_id", "").strip() active_doc_state = form_data.get("active_doc_state", "").strip().casefold() + # Scoped photo_editor agent context: the Quick Edit popup (and any + # image-edit chat turn) may carry the open image-editor session id and a + # canvas snapshot so the editor tool family can drive it in-process. + # The id is quoted into a system message, so accept only an + # identifier; any other value is treated as no open editor. + editor_session = str( + form_data.get("editor_session") + or form_data.get("editor_session_id") + or "" + ).strip() + if not _EDITOR_SESSION_RE.fullmatch(editor_session): + editor_session = "" + photo_editor_requested = ( + str(form_data.get("photo_editor") or "").strip().lower() + in {"1", "true", "yes", "on"} + or bool(editor_session) + ) logger.info( "[doc-inject] chat_mode=%s, active_doc_id=%r, active_doc_state=%r", chat_mode, active_doc_id, active_doc_state, @@ -3270,7 +3290,34 @@ def setup_chat_routes( active_document=bool(active_doc), workspace=bool(workspace), image_attachment=any(str(a.get('mime') or '').startswith('image/') for a in (ctx.preprocessed.attachment_meta or [])), ) if _use_turn_contract else frozenset() - if 'image_editing' in _turn_capabilities and ctx.preprocessed.attachment_meta: + # Scoped photo_editor family: offered ONLY for an explicit editor + # request (Quick Edit popup / editor session) or an image-edit chat + # turn carrying an image. When active the turn offers just the editor + # tools plus ask_user, never the general tool list. + _photo_editor_active = bool(_use_turn_contract) and ( + photo_editor_requested + or ( + "image_editing" in _turn_capabilities + and bool(ctx.preprocessed.attachment_meta) + ) + ) + if _photo_editor_active: + if editor_session: + editor_context = {'role': 'system', 'content': + 'The image editor is open (session ' + editor_session + '). Use run_editor_command, ' + 'plan_edit and get_editor_screenshot to inspect and change the current canvas, and ' + 'ai_edit_image for model-based edits. Apply the user\'s requested edit to the open ' + 'editor and confirm the result with a screenshot.'} + else: + editor_context = {'role': 'system', 'content': + 'The user attached an image but no image editor is open, so the editor tools cannot ' + 'reach the canvas. Do not call editor tools. Reply briefly telling the user to open ' + 'the attached image in the editor (Quick Edit) and repeat the request; the editor ' + 'tools will then apply it.'} + ctx.messages.insert(0, editor_context) + if foreground_policy.enabled: + getattr(ctx, 'route_messages', ctx.messages).insert(0, dict(editor_context)) + elif 'image_editing' in _turn_capabilities and ctx.preprocessed.attachment_meta: image_refs = ['odysseus://attachment/' + str(a['id']) for a in ctx.preprocessed.attachment_meta if a.get('id') and str(a.get('mime') or '').startswith('image/')] if image_refs: @@ -3307,6 +3354,11 @@ def setup_chat_routes( # pasted revision request. This only offers permitted schemas; # it never requires or performs a document mutation. _turn_capabilities = _turn_capabilities | {"documents"} + # Scoped photo_editor family (see above): when active the turn offers + # just the editor tools plus ask_user, never the general tool list. + if _photo_editor_active: + _turn_capabilities = frozenset({"photo_editor"}) + _active_turn_capabilities = _turn_capabilities # Same decision that prepared the turn; it only stamps the contract # inside the agent-contract branch below. _clean_v3_preview = _compact_preview_turn @@ -3474,7 +3526,7 @@ def setup_chat_routes( if not _privs.get("can_use_documents", True): disabled_tools.update({"manage_documents", "create_document", "edit_document", "update_document", "suggest_document"}) if not _privs.get("can_generate_images", True): - disabled_tools.update({"generate_image", "edit_image"}) + disabled_tools.update({"generate_image", "edit_image", "ai_edit_image"}) if not _privs.get("can_manage_memory", True): disabled_tools.update({"manage_memory", "manage_skills"}) if not _privs.get("can_use_research", True): @@ -3556,9 +3608,22 @@ def setup_chat_routes( "document_editor": not { "manage_documents", "create_document", "edit_document", "update_document", }.issubset(disabled_tools), + # A photo-editing chat turn can only reach a canvas when the image + # editor is actually open. The compact runtime uses this to reply + # (open the image in the editor) instead of looping on editor tools + # that have no canvas to drive. + "photo_editor_open": bool(editor_session), } # Capture permission state before schema selection/reconciliation. # Only deterministic request intent supplies grants, never inventory. + # The strict router (spec) surface picks its tools in src/spec_router.py + # AFTER authority is built, so its admission is keyed off the surface. + try: + _spec_surface = _configured_model_tool_surface_raw( + getattr(sess, "endpoint_url", ""), getattr(sess, "model", ""), _user + ) == "spec" + except Exception: + _spec_surface = False _request_authority = request_authority_for_http( request, message, owner=_user, session_id=session, workspace=workspace, history=_turn_history, policy=tool_policy, @@ -3566,9 +3631,9 @@ def setup_chat_routes( active_document=bool(active_doc), image_attachment=any(str(a.get('mime') or '').startswith('image/') for a in (ctx.preprocessed.attachment_meta or [])), - capabilities=({'search_browser'} if ( + capabilities=(({"photo_editor"} if _photo_editor_active else set()) | ({'search_browser'} if ( _explicit_browser_intent or _external_discovery_intent - ) else ()), + ) else set())), ) if exact_tool_approval is not None: from src.agent_runtime.authority import RequestAuthority @@ -3758,7 +3823,9 @@ def setup_chat_routes( # personal manager; core shell/Web tools are not task # fallbacks. Other turns retain warm and workspace tools. extra_tools=( - frozenset() + frozenset({"ask_user"}) + if _active_turn_capabilities == frozenset({"photo_editor"}) + else frozenset() if _exact_selected_native_chain or _active_turn_capabilities == frozenset({"tasks"}) or ( _active_turn_capabilities in ( frozenset({"transcription"}), frozenset({"ocr"}), @@ -3960,7 +4027,7 @@ def setup_chat_routes( _heartbeat_counter = 0 _last_progress = {} _sent_avg = False - while True: + while not _shutdown.is_shutting_down(): status = research_handler.get_status(session) if not status or status["status"] != "running": break @@ -4222,8 +4289,6 @@ def setup_chat_routes( # indicator, but don't fold them into the saved # reply (mirrors the rewrite path below). if data.get("thinking"): - if thinking_mode == "off": - continue thinking_response += data["delta"] else: full_response += data["delta"] @@ -4667,8 +4732,6 @@ def setup_chat_routes( # Forward them for the live indicator, but keep # them out of the saved reply (same as chat mode). if data.get("thinking"): - if thinking_mode == "off": - continue thinking_response += data["delta"] else: full_response = _render_state.content @@ -4852,6 +4915,25 @@ def setup_chat_routes( _terminal_saved = True if _saved_id: yield f'data: {json.dumps(_render_state.message_saved(_saved_id))}\n\n' + run_post_response_tasks( + sess, session_manager, session, message, _saveable_no_tool_response, + _metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager, + incognito=incognito, compare_mode=compare_mode, + character_name=ctx.preset.character_name, + agent_rounds=_agent_rounds, + agent_tool_calls=_agent_tool_calls, + skills_manager=skills_manager, + owner=_user, + extract_skills=( + user_requested_agent + and not tool_approval_continuation + ), + allow_background_extraction=_post_response_extraction_allowed( + tools_blocked=tool_policy.block_all_tool_calls, + tool_approval_continuation=tool_approval_continuation, + client_runtime_context=client_runtime_context, + ), + ) yield f'data: {json.dumps(_metrics_event)}\n\n' except json.JSONDecodeError: yield chunk @@ -4962,6 +5044,8 @@ def setup_chat_routes( raises before reaching a mode-specific finally block.""" try: async for chunk in stream_with_save(): + if _shutdown.is_shutting_down(): + break yield chunk finally: _active_streams.pop(session, None) @@ -5056,7 +5140,12 @@ def setup_chat_routes( if rec is None: if agent_runs.is_active(session_id): return {"status": "streaming", "detached": True} - raise HTTPException(404, "No active stream for this session") + # No active stream: report an explicit idle state with HTTP 200. + # A 404 here made every session load look like a failed request in + # the browser console. The client distinguishes "no stream" by the + # body (status != "streaming"), and still tolerates a 404 from an + # older server, so this stays backward compatible. + return {"status": "idle"} return rec # ------------------------------------------------------------------ # @@ -5155,6 +5244,8 @@ def setup_chat_routes( max_tokens=0, tools=None, ): + if _shutdown.is_shutting_down(): + break if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: data = json.loads(chunk[6:]) diff --git a/routes/document/document_routes.py b/routes/document/document_routes.py index 9a7743e8a..2b987e140 100644 --- a/routes/document/document_routes.py +++ b/routes/document/document_routes.py @@ -1185,7 +1185,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: async def ai_tidy_documents(request: Request) -> Dict[str, Any]: """Use AI to judge if documents are junk/test/accidental, then delete them. Caches verdicts so previously-reviewed docs are skipped.""" - from src.task_endpoint import resolve_task_endpoint + from src.task_endpoint import resolve_task_endpoint, task_llm_call_async from src.endpoint_resolver import resolve_endpoint from src.llm_core import llm_call_async @@ -1228,13 +1228,13 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: + "\n".join(doc_list) ) - response = await llm_call_async( - url, model, + response = await task_llm_call_async( [{"role": "system", "content": "You classify documents as junk or keep. Respond only with a JSON array."}, {"role": "user", "content": prompt}], + fallback_url=url, fallback_model=model, fallback_headers=headers, + owner=user or None, temperature=0.1, max_tokens=200, - headers=headers, timeout=30, ) diff --git a/routes/editor_agent_routes.py b/routes/editor_agent_routes.py new file mode 100644 index 000000000..717c354e9 --- /dev/null +++ b/routes/editor_agent_routes.py @@ -0,0 +1,2018 @@ +"""Editor agent routes: natural-language planning for the image editor. + +Provides: + POST /api/editor/plan -> {reply, planner, steps:[{op,params}], done?} + POST /api/editor/bridge/command -> enqueue validated ops for the open editor + GET /api/editor/bridge/events -> SSE stream of queued commands (browser subscriber) + POST /api/editor/bridge/result -> browser posts results/screenshots back + POST /api/editor/bridge/await -> wait for a queued command's result + +The browser editor (static/js/editor/nl-edit.js) posts {prompt, state, mode, screenshot} +and consumes {reply, planner, steps} or {done:true}. When mode == "check" and the model +reports completion, the route returns {"done": true} to stop the client's self-check loop. + +The planner is wired to the app's configured chat model (vision-capable when a screenshot +is supplied) via set_editor_planner(); a deterministic keyword planner is the fallback so +the editor always gets a usable plan. +""" +from __future__ import annotations + +import asyncio +import inspect +import json +import logging +import math +import re +import uuid +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from src import editor_style_guide as style_guide +from src import editor_agent_model as editor_agent_model +from src import shutdown as _shutdown + +logger = logging.getLogger(__name__) + +# ── Op catalog (kept in sync with static/js/editor/editor-control.js) ── +OP_CATALOG = [ + "document.info", "document.importImage", + "document.crop", "document.resizeCanvas", "document.savePsd", + "layers.list", "layers.select", "layers.addRaster", "layers.placeImage", "layers.addAdjustment", + "layers.duplicate", "layers.delete", "layers.setOpacity", "layers.setBlend", + "layers.group", "layers.ungroup", "layers.addStyle", + "ai.generativeExpand", + "selection.selectAll", "selection.deselect", "selection.invert", + "selection.rectangle", "selection.ellipse", + "selection.subject", "selection.sky", "selection.background", "selection.samText", + "selection.colorRange", "selection.maskBounds", "selection.exportMask", + "adjustments.apply", "adjustments.applyMasked", + "filters.apply", "filters.applyMasked", + "transform.start", "transform.apply", "transform.cancel", + "text.add", + "history.mark", "history.collapse", "history.undo", "history.redo", + "export.png", "export.gallery", "screenshot", +] +# Retouch/selection ops are defined once in the shared style guide so the +# server planner and the browser fallback cannot drift; a plan naming one of +# these would otherwise be filtered out of the catalog. +for _extra_op in style_guide.NEW_OP_NAMES: + if _extra_op not in OP_CATALOG: + OP_CATALOG.append(_extra_op) +del _extra_op +_OP_SET = frozenset(OP_CATALOG) + +# Canonical argument shapes advertised to the model so it stops guessing. The +# browser client tolerates loose forms as a safety net, but the model should +# emit exactly these. +_ADJUSTMENT_TYPES = ( + "brightness-contrast {brightness,contrast} (multipliers, 1=neutral), " + "exposure {exposure,offset,gamma}, white-balance {temperature,tint}, " + "hue-saturation {hue,saturation,lightness}, vibrance {vibrance}, black-white, " + "shadows-highlights {shadows,highlights}, levels, curves, color-balance, " + "photo-filter {color,density,preserve_luminosity}, selective-color, gradient-map, " + "invert, posterize, threshold, channel-mixer, color-lookup" +) +_FILTER_IDS = ( + "blur-gaussian, blur-motion, blur-zoom, sharpen, sharpen-edges, unsharp-mask, " + "noise-add, noise-despeckle, pixelate-mosaic, pixelate-crystallize, stylize-emboss, " + "stylize-find-edges, stylize-solarize, distort-twirl, distort-pinch, distort-spherize, " + "distort-ripple, render-clouds, clarity {amount,radius}, vignette {amount,feather}" +) +_CANONICAL_GUIDANCE = ( + "Canonical op shapes (use exactly these, never invent keys or names): " + "adjustments.apply / adjustments.applyMasked params = " + "{\"type\": , \"params\": {...}}; " + "filters.apply / filters.applyMasked params = {\"id\": }. " + "Never nest a numeric bag under {\"adjustments\": {...}} with no type, and never invent " + "adjustment types or filter ids. \"clarity\" (params amount,radius) and \"vignette\" " + "(params amount,feather) are real filters — use them for local-contrast and framing. " + "Canvas: document.crop {x,y,width,height}, document.resizeCanvas {width,height,anchor}, " + "ai.generativeExpand {width,height,anchor,prompt,mode:'instruction',feather}. Text/styles: " + "text.add {text,x,y,color,size}, layers.addStyle {type,params}. Export: document.savePsd. " + "ai_edit_image takes a mode: 'instruction' (default; whole image, no mask, prompt names " + "what stays unchanged) for background/scene/season/style changes, adding an object to an " + "existing photo, and removing diffuse distractions; 'masked' (tight selection first) ONLY " + "for explicit regional removal of a clearly identified object. generate_image is ONLY for " + "creating a brand-new image." +) +# The shared style guide is the single source of truth for intent routing, +# calibrated strengths and the low-key/temperature/portrait rules. Both the +# server planner and the browser fallback read it (the client fetches +# GET /api/editor/style-guide). +_STYLE_GUIDANCE = style_guide.guidance() + +# EDITOR_PLANNER: async (messages: list[dict], owner: Optional[str]) -> str | dict. Set by app.py. +_EDITOR_PLANNER: Optional[Callable[[List[dict], Optional[str]], Any]] = None +_MAX_STEPS = 32 +_QUEUE_MAX = 64 +# Generative editor steps (SAM mask + local diffusion) take 30-110s each, so +# bridge waits and the agent's per-step wait must outlast the provider. +_BRIDGE_DEFAULT_TIMEOUT = 330.0 + +# ── Strict JSON ops schema (structured outputs / grammar) ── +# The planner (the local Qwen3.5-9B at :8097 and every fallback) is constrained +# to this schema so it can only emit valid editor ops. Allowed ops: +# * the deterministic editor ops (OP_CATALOG), +# * generate_edit -> image model (instruction, optional mask) [=ai_edit_image] +# * select_object -> SAM mask by description (seneca /v1/images/mask) +# [=selection.samText] +# * ask_user -> ask 2-3 concrete choices (never a silent tone default). +# The model-facing aliases are translated to the internal execution ops below, +# so the browser client and the executor keep their existing op vocabulary. +_MODEL_PLAN_OPS = tuple(dict.fromkeys( + list(OP_CATALOG) + ["generate_edit", "select_object", "ai_edit_image", + "generate_image", "ask_user"] +)) +_ALLOWED_PLAN_OPS = frozenset(OP_CATALOG) | {"ai_edit_image", "generate_image"} +_PLAN_OP_ALIASES = { + "generate_edit": "ai_edit_image", # image model: instruction + optional mask + "select_object": "selection.samText", # SAM mask by description +} +# Inner op values the ai_edit_image tool accepts (src/agent_tools/editor_tools.py +# `_AI_OPS`). A model-authored ai_edit_image step frequently omits the inner op or +# invents one (e.g. "restyle"), which the tool then rejects. Normalise it so +# instruction/restyle edits become op "generative" and masked region edits become +# "inpaint". +_AI_IMAGE_INNER_OPS = frozenset( + {"remove-bg", "upscale", "denoise", "enhance-face", "inpaint", "generative"} +) + + +def _normalize_ai_step(params: Any) -> Dict[str, Any]: + """Repair an ai_edit_image step's inner op so it always runs (restyle -> + generative, masked -> inpaint).""" + params = dict(params) if isinstance(params, dict) else {} + mode = str(params.get("mode") or "").strip().lower() + inner = str(params.get("op") or "").strip().lower() + if inner not in _AI_IMAGE_INNER_OPS: + inner = "inpaint" if mode == "masked" else "generative" + params["op"] = inner + if inner in ("inpaint", "generative") and not mode: + params["mode"] = "masked" if inner == "inpaint" else "instruction" + return params +PLAN_SCHEMA: Dict[str, Any] = { + "type": "object", + "properties": { + "reply": {"type": "string"}, + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": list(_MODEL_PLAN_OPS), + "description": ( + "generate_edit = AI image model edit (params {prompt, " + "mode:'instruction'|'masked'}; do NOT add an inner 'op' — " + "instruction/restyle edits run as the image model's " + "'generative' op, masked region edits as 'inpaint'). " + "select_object = SAM mask by text (params {text}). " + "ask_user = ask with 2-3 choices (params {question, options})." + ), + }, + "params": {"type": "object"}, + }, + "required": ["op"], + }, + }, + "ask_user": { + "type": "object", + "properties": { + "question": {"type": "string"}, + "options": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["question", "options"], + }, + "done": {"type": "boolean"}, + }, + "required": ["reply", "steps"], +} +PLAN_RESPONSE_FORMAT: Dict[str, Any] = { + "type": "json_schema", + "json_schema": {"name": "editor_plan", "schema": PLAN_SCHEMA}, +} + + +def _normalize_ask_user(payload: Any) -> Optional[Dict[str, Any]]: + """Validate an ask_user payload: one question and 2-3 concrete choices.""" + if not isinstance(payload, dict): + return None + question = str(payload.get("question") or "").strip() + raw = payload.get("options") + labels: List[str] = [] + for opt in (raw or []): + if isinstance(opt, str): + lbl = opt.strip() + elif isinstance(opt, dict): + lbl = str(opt.get("label") or "").strip() + else: + lbl = "" + if lbl: + labels.append(lbl) + labels = labels[:3] + if not question or len(labels) < 2: + return None + # Match the shape the ask_user tool + the editor client expect: option + # objects with a label (and an optional description). + return { + "question": question, + "options": [{"label": lbl, "description": ""} for lbl in labels], + "multi": False, + } + + +def _validate_plan(data: Any) -> Optional[Dict[str, Any]]: + """Validate a raw planner plan against the strict ops schema. + + Returns a normalized plan (aliases translated, ops checked against the + allowed set, ask_user normalized to 2-3 choices) or None when the plan is + invalid or empty. Callers turn None into an ask_user plan — never a silent + tone default. + """ + if not isinstance(data, dict): + return None + reply = str(data.get("reply") or "") + ask_user = _normalize_ask_user(data.get("ask_user")) + raw_steps = data.get("steps") + if raw_steps is None: + raw_steps = [] + if not isinstance(raw_steps, list): + return None + steps: List[Dict[str, Any]] = [] + for s in raw_steps[:_MAX_STEPS]: + if not isinstance(s, dict): + continue # drop a malformed step rather than rejecting the whole plan + op = str(s.get("op") or "") + if op == "ask_user": + au = _normalize_ask_user(s.get("params")) + if au is not None and ask_user is None: + ask_user = au + continue + op = _PLAN_OP_ALIASES.get(op, op) + if op not in _ALLOWED_PLAN_OPS: + continue # drop a single unknown/hallucinated op; keep the valid steps + params = s.get("params") if isinstance(s.get("params"), dict) else {} + if op == "ai_edit_image": + params = _normalize_ai_step(params) + steps.append({"op": op, "params": params}) + # Guardrails apply to model plans too (no stacked lifts, no brighten after + # retouch.skin, no global adjustment after a masked recolour). + steps = style_guide.enforce_guards(steps) + done = bool(data.get("done")) + if not steps and not done and ask_user is None: + return None + out: Dict[str, Any] = {"reply": reply, "planner": "llm", "steps": steps} + if ask_user is not None: + out["ask_user"] = ask_user + if done: + out["done"] = True + return out + +# Generative edits (remove/add/replace content) require an image provider. When +# none is available the agent must stop and say so plainly rather than apply a +# deterministic approximation that silently does the wrong thing. +try: + from src.ai_interaction import IMAGE_PROVIDER_REQUIRED_MESSAGE as _PROVIDER_REQUIRED_MESSAGE +except Exception: # pragma: no cover - defensive + _PROVIDER_REQUIRED_MESSAGE = ( + "Removing/adding objects needs an image generation provider; enable one in " + "Settings > Models." + ) + +_AI_CONTENT_OPS = frozenset({"ai_edit_image", "generate_image"}) + +# Model ids that are editing/inpainting models rather than text-to-image ones. +_EDIT_MODEL_RE = re.compile(r"edit|inpaint|fill|kontext|controlnet|removal|qwen-image-edit", re.I) + +# Content nouns that signal a generative object edit rather than a tone/filter +# tweak. Kept narrow so "make it black and white" / "add a vignette" stay +# deterministic. +_GEN_OBJECTS = ( + r"(?:persons?|people|men|man|women|woman|guys?|girls?|boys?|someone|somebody|" + r"objects?|car|cars|sign|signs|logos?|watermarks?|trees?|buildings?|" + r"reflections?|animals?|dogs?|cats?|birds?|hands?|fingers?|moons?|suns?|clouds?|" + r"stars?|fire|smoke|snow|rain|flowers?|mountains?|planets?|balloons?|rockets?|hats?|" + r"glasses|sunglasses|shades|beard|beards|stickers?|frames?|borders?|" + r"rainbows?|wildflowers?|sunflowers?|tulips?|daisies?|leaves|pumpkins?|lanterns?|" + r"butterflies?|bees?|boats?|sailboats?|kites?|umbrellas?)" +) +_REMOVE_RE = re.compile( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out|photoshop\s+out|paint\s+out)\b" + r"[^.!?\n]{0,60}?\b" + _GEN_OBJECTS + r"\b", + re.I, +) +_ADD_RE = re.compile( + r"\b(?:add|insert|put|place|draw|paint|create|generate)\b" + r"[^.!?\n]{0,40}?\b" + _GEN_OBJECTS + r"\b", + re.I, +) +_REPLACE_RE = re.compile( + r"\b(?:replace|swap|substitute)\b|\b(?:change|turn)\b[^.!?\n]{0,40}?\b(?:to|into)\b", + re.I, +) +# Removing diffuse clutter ("remove the distractions") is generative too, but +# the specific-noun _REMOVE_RE above does not catch it. +_REMOVE_CLEANUP_RE = re.compile( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out|clean\s+up|declutter|clear)\b" + r"[^.!?\n]{0,40}?\b(?:distractions?|clutter|junk|mess|noise|" + r"unwanted\s+(?:objects?|items?|things?)|stuff|things|blemishes?|spots?)\b", + re.I, +) + +# Changing the whole background/scene (or placing the subject somewhere new) +# is a generative replacement, not the local remove-bg op. "make the +# background nicer/better" is included; plain "blur the background" is not. +_BG_RE = re.compile( + r"\b(?:replace|change|swap|put|place|set|make|improve|enhance|beautify|upgrade)\b" + r"[^.!?\n]{0,40}?\b(?:background|backdrop|scene)\b" + r"|\b(?:put|place)\b[^.!?\n]{0,30}?\b(?:on|in|at)\s+(?:a|an|the)\s+" + r"(?:beach|street|mountains?|forest|city|park|ocean|sea|desert|room|stage|field|snow|" + r"night|moon|mars|space|island|jungle)\b", + re.I, +) +# Season/weather changes and an explicit "generative fill" change scene +# content (not just tone), so they must be a generative edit, never a +# colour adjustment/filter step. +_SEASON_WEATHER_RE = re.compile( + r"\b(?:snow(?:y|ing|fall)?|winter|autumn(?:al)?|fall\s+(?:leaves|foliage)|spring|" + r"rain(?:y|ing)?|fog(?:gy)?|mist(?:y)?|storm(?:y)?|overcast)\b", + re.I, +) +_GENERATIVE_FILL_RE = re.compile(r"\bgenerative(?:\s+fill)?\b", re.I) +# Whole-image artistic RESTYLE requests (turn the photo into a painting, anime, +# pixel art, ...). These are a model instruction edit over the whole frame, not a +# local adjustment. The client has a short fast-path regex; the server must +# generalise so unknown restyle phrasings still reach the model. +_RESTYLE_RE = re.compile( + r"\b(?:re[\s-]?style|painting|painterly|water[\s-]?colou?r|oil\s+paint(?:ing)?|" + r"acrylic|gouache|pastel|sketch|pencil|charcoal|ink\s+drawing|line\s+art|comic|" + r"cartoon|anime|manga|illustration|3d\s+render|cgi|claymation|clay|pixel\s+art|" + r"voxel|low[\s-]?poly|mosaic|stained\s+glass|pop\s+art|art\s+nouveau|ukiyo[\s-]?e|" + r"surreal|fantasy\s+art|steampunk|cyberpunk|vaporwave|synthwave|graffiti|woodcut|" + r"linocut|etching|lithograph|impressionist|cubist|abstract|doodle|vector\s+art|" + r"neon\s+sign|psychedelic|gothic|comic[\s-]?book|storybook)\b", + re.I, +) + +# Text/title/PSD/format requests are deterministic edits, never generative. +_TEXT_RE = re.compile( + r"\b(?:title|caption|subtitle|label|heading|watermark|text|word(?:ing)?)\b", re.I, +) +_TEXT_ADD_RE = re.compile(r"\b(?:add|insert|put|place|write|overlay|create|set)\b", re.I) +_STYLE_RE = re.compile( + r"\b(drop[\s-]?shadow|outer[\s-]?glow|glow|stroke|outline|emboss|bevel)\b", re.I, +) +_PSD_RE = re.compile(r"\b(?:psd|photoshop file|layered file)\b", re.I) + +# Aspect -> exact output size. Instagram/format intents crop to the aspect and +# then resize to this size (then optionally add a title). +_ASPECT_TARGETS = ( + (re.compile(r"\b(?:1080\s*[x×]\s*1350|4\s*:\s*5|four[\s-]?fifths?|instagram)\b", re.I), (1080, 1350)), + (re.compile(r"\b(?:1080\s*[x×]\s*1080|1\s*:\s*1|square)\b", re.I), (1080, 1080)), + (re.compile(r"\b(?:1080\s*[x×]\s*1920|9\s*:\s*16|story|reel|tiktok|vertical)\b", re.I), (1080, 1920)), + (re.compile(r"\b(?:1920\s*[x×]\s*1080|16\s*:\s*9|widescreen|landscape video)\b", re.I), (1920, 1080)), +) +_DIM_RE = re.compile(r"\b(\d{3,5})\s*[x×]\s*(\d{3,5})\b") + + +def _is_text_request(text: str) -> bool: + return bool(_TEXT_RE.search(text) and _TEXT_ADD_RE.search(text)) + + +def _format_target(text: str) -> Optional[Tuple[int, int]]: + for pattern, dims in _ASPECT_TARGETS: + if pattern.search(text): + return dims + m = _DIM_RE.search(text) + if m: + try: + return (int(m.group(1)), int(m.group(2))) + except Exception: + return None + return None + + +def _extend_requested(text: str) -> bool: + if re.search(r"\b(?:extend|expand|wider|taller|letterbox)\b", text, re.I): + return True + return bool(re.search( + r"\badd\b[^.!?\n]{0,25}?\b(?:canvas|border|space|margin)\b", text, re.I, + )) + + +def _generative_kind(prompt: str) -> Optional[str]: + """Classify a prompt as a generative content edit: 'remove'/'add'/'replace'. + + Returns None for tone/colour/filter/text/selection edits that deterministic + editor ops handle correctly. 'remove the background' is deliberately not + matched (that is the local remove-bg op). + """ + text = str(prompt or "") + if not text.strip(): + return None + # Text/title, PSD and format requests are deterministic — never generative. + if _is_text_request(text) or _PSD_RE.search(text) or _format_target(text): + return None + # An explicit brand-new image ("create/generate/make a new image/picture/logo + # ...") is a text-to-image generation, never an edit of the current photo. + if _is_new_image_request(text): + return "add" + # Replacing the background/scene (or putting the subject somewhere new) is + # a generative replacement even though it is phrased like a background op. + if _BG_RE.search(text): + return "replace" + if re.search( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of)\s+(?:the\s+)?background\b", + text, + re.I, + ): + return None + # Season/weather changes and explicit "generative fill" change content, not + # tone. Only treat them as generative when the user is not asking to REMOVE + # that element ("remove the rain" stays a removal). + if _GENERATIVE_FILL_RE.search(text) or _SEASON_WEATHER_RE.search(text): + if not re.search( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out)\b", text, re.I + ): + return "replace" + if _RESTYLE_RE.search(text) and not _is_new_image_request(text): + # A whole-image artistic restyle is a model instruction edit (keep the + # subject/composition), never a local tone/colour adjustment. + return "restyle" + if _REPLACE_RE.search(text): + return "replace" + if _ADD_RE.search(text): + return "add" + if _REMOVE_RE.search(text) or _REMOVE_CLEANUP_RE.search(text): + return "remove" + return None + + +_NEW_IMAGE_RE = re.compile( + r"\b(?:create|generate|make|produce|draw|design)\b[^.!?\n]{0,30}?\b" + r"(?:new\s+)?(?:image|picture|photo|illustration|artwork|graphic|logo|icon|wallpaper|poster)\b", + re.I, +) +_EDIT_CONTEXT_RE = re.compile( + r"\b(?:this|that|these|it|the\s+(?:photo|image|picture)|attached|from\s+this|" + r"of\s+(?:this|the\s+photo)|edit|in\s+(?:this|the)\s+photo)\b", + re.I, +) +_KEEP_CLAUSE_RE = re.compile(r"\bkeep\b|\bunchanged\b|\bpreserve\b|\bsame\b|\bidentical\b", re.I) + + +def _is_new_image_request(text: str) -> bool: + """True only for a from-scratch image generation (not an edit of an existing photo).""" + if not _NEW_IMAGE_RE.search(text): + return False + return not _EDIT_CONTEXT_RE.search(text) + + +def _instruction_prompt(text: str) -> str: + """Append a keep-unchanged clause so instruction edits preserve the subject.""" + t = str(text or "").strip() + if not t: + return t + if _KEEP_CLAUSE_RE.search(t): + return t + return ( + t.rstrip(".") + + ". Keep the main subject and everything else in the image exactly the same." + ) + + +def _restyle_prompt(text: str) -> str: + """Whole-image artistic restyle instruction (subject/composition preserved).""" + t = str(text or "").strip() + if t: + base = ( + "Restyle the entire image as requested, applying the artistic style " + "across the whole frame: " + t + ) + else: + base = "Restyle the entire image in a new artistic style." + return _instruction_prompt(base) + + +def _generative_mode(kind: str, prompt: str) -> str: + """'masked' only for an explicit regional removal of a clearly identified object. + + Everything else generative (background/season/style replacement, adding an + object to the scene, removing diffuse distractions) uses 'instruction'. + """ + if kind == "remove": + text = str(prompt or "") + if _REMOVE_RE.search(text) and not _REMOVE_CLEANUP_RE.search(text): + return "masked" + return "instruction" + + +def _removal_prompt(text: str) -> str: + """A more explicit diffuse-removal instruction that names common clutter.""" + t = str(text or "").strip() or "Remove the distracting clutter from the photo" + return ( + t.rstrip(". ") + + ". Remove any distracting clutter such as roads, guardrails, utility poles, " + "signs, wires and debris, and fill the cleared area with the surrounding " + "landscape so it looks natural." + ) + + +# Rich, scene-specific descriptions so a background replacement produces a +# convincing scene (Qwen-Image-Edit is an instruction model and a bare "a beach" +# tends to yield a flat blue backdrop). Keyed by keywords in the request. +_SCENE_HINTS = ( + (r"beach|shore|seaside|coast|tropical", + "a sunny tropical beach", + "golden sand, turquoise sea with gentle waves, a clear blue sky with soft " + "clouds, and natural daylight"), + (r"mountain|alps|peak|valley", + "a scenic mountain landscape", + "distant peaks, natural daylight and soft haze"), + (r"forest|woods|jungle", + "a lush green forest", + "dappled natural daylight and depth"), + (r"city|street|urban|downtown", + "a realistic city street", + "recognisable city buildings and depth of field with natural perspective"), + (r"field|meadow|grass", + "an open grassy field", + "natural daylight and a distant horizon"), + (r"snow|winter", + "a snow-covered landscape", + "cold, soft daylight and a pale sky"), + (r"desert|dunes", + "a warm desert", + "sand dunes and a clear sky"), + (r"studio|plain|neutral|solid", + "a clean neutral studio backdrop", + "soft, even lighting"), +) +_SCENE_DEFAULT = ( + "a realistic natural background", + "matching the subject's lighting, perspective and shadows", +) +# Season/weather scene descriptions ("make it snowy/autumn"). +_SEASON_HINTS = ( + (r"snow|winter|snowy", "a snowy winter scene", + "fresh snow covering the ground and surfaces, soft cold daylight, a pale overcast sky"), + (r"autumn|autumnal|fall", "an autumn scene", + "vivid orange and golden foliage, warm low sunlight and fallen leaves"), + (r"spring", "a bright spring scene", + "fresh green foliage and blossoms under bright natural daylight"), + (r"rain|storm|overcast", "a rainy overcast scene", + "wet reflective surfaces, raindrops and a dramatic overcast sky"), + (r"fog|mist", "a foggy scene", + "soft drifting mist and diffuse light"), + (r"sky|clouds", "a new sky", + "realistic clouds and natural lighting matching the scene"), +) + + +def _scene_prompt(text: str) -> str: + """Deterministic, templated instruction for a background/scene change. + + Qwen-Image-Edit is an instruction model and a bare scene noun tends to yield + a flat backdrop, so the scene is expanded into concrete detail. The result is + passed to the provider unmodified. + """ + t = str(text or "").strip() + if not t: + return t + if _BG_RE.search(t): + scene, details = _SCENE_DEFAULT + for pattern, scene_name, description in _SCENE_HINTS: + if re.search(pattern, t, re.I): + scene, details = scene_name, description + break + return ( + f"Replace the entire background behind the person with {scene}: {details}. " + "Keep the person exactly the same." + ) + if _SEASON_WEATHER_RE.search(t) or _GENERATIVE_FILL_RE.search(t): + scene, details = _SCENE_DEFAULT + for pattern, scene_name, description in _SEASON_HINTS: + if re.search(pattern, t, re.I): + scene, details = scene_name, description + break + return ( + f"Change the scene to {scene}: {details}. " + "Keep the main subject and everything else in the image exactly the same." + ) + return t + + +def _generative_step(kind: str, prompt: str) -> Dict[str, Any]: + """The AI step a generative intent must use instead of deterministic ops.""" + instruction = str(prompt or "").strip() + if kind == "restyle": + return { + "op": "ai_edit_image", + "params": {"op": "generative", "mode": "instruction", + "prompt": _restyle_prompt(instruction), "steps": 8}, + } + if kind == "add": + # A brand-new image is the only case for generate_image; adding an object + # to an existing photo is an instruction edit of the whole image. + if _is_new_image_request(prompt): + return {"op": "generate_image", "params": {"prompt": instruction}} + return { + "op": "ai_edit_image", + "params": { + "op": "generative", + "mode": "instruction", + "prompt": _instruction_prompt(instruction), + "steps": 8, + }, + } + mode = _generative_mode(kind, prompt) + op = "inpaint" if kind == "remove" else "generative" + if mode == "instruction": + if kind == "remove": + base = _removal_prompt(instruction) + else: + base = _scene_prompt(instruction) + params: Dict[str, Any] = {"op": op, "mode": "instruction", "prompt": _instruction_prompt(base)} + else: + params = {"op": op, "mode": "masked", "prompt": instruction} + if kind in ("remove", "replace"): + # Removal and background-replacement edits need more sampler steps than + # the server's default of 4. + params["steps"] = 8 + return {"op": "ai_edit_image", "params": params} + + +_SKY_ADD_RE = re.compile( + r"\b(?:sky|moons?|suns?|sunset|sunrise|clouds?|stars?|night|aurora|stars?)\b", + re.I, +) +_REMOVE_NOUN_RE = re.compile( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out|clean\s+up|declutter|clear)\b" + r"[^.!?\n]{0,60}?\b(" + + _GEN_OBJECTS + + r"|distractions?|clutter|junk|mess|noise|stuff|things)\b", + re.I, +) + + +def _selection_step_for(kind: str, prompt: str) -> Optional[Dict[str, Any]]: + """A deterministic selection step that yields the mask a MASKED ai_edit_image needs. + + Instruction-mode edits operate on the whole image and need no mask, so this + returns None for them. Masked mode is only used for an explicit regional + object removal; the selection targets that object so ``selection.exportMask`` + has a tight region to export. + """ + text = str(prompt or "") + if kind == "remove" and _generative_mode(kind, text) == "masked": + m = _REMOVE_NOUN_RE.search(text) + label = (m.group(1).lower() if m else "person") + if label.endswith("s"): + label = label[:-1] + if label in ("people",): + label = "person" + return {"op": "selection.subject", "params": {"label": label}} + return None + + +def _generative_steps(kind: str, prompt: str) -> List[Dict[str, Any]]: + """The selection step (when needed) followed by the AI content step.""" + steps: List[Dict[str, Any]] = [] + selection = _selection_step_for(kind, prompt) + if selection: + steps.append(selection) + steps.append(_generative_step(kind, prompt)) + return steps + + +def _plan_uses_ai_content(steps: Any) -> bool: + return any(isinstance(s, dict) and s.get("op") in _AI_CONTENT_OPS for s in (steps or [])) + + +def _ensure_generative(prompt: str, steps: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Force a generative intent onto an AI content op, never a substitute. + + If the prompt is a generative content edit (remove/add/replace) but the + plan has no ai_edit_image/generate_image step, replace the plan with the + correct AI step so the agent cannot silently do an unrelated deterministic + edit. Non-generative prompts are returned unchanged. + """ + kind = _generative_kind(prompt) + if not kind: + return steps + # Background/scene replacement is deterministic: always use the templated + # scene prompt so a model rewrite cannot drop the scene details (a model + # ai_edit_image step would otherwise pass with its own vague prompt). + if kind == "replace" and ( + _BG_RE.search(str(prompt or "")) + or _SEASON_WEATHER_RE.search(str(prompt or "")) + or _GENERATIVE_FILL_RE.search(str(prompt or "")) + ): + return _generative_steps(kind, prompt) + # For a restyle (and any other generative intent) the planner is expected to + # choose generate_edit itself; the _RESTYLE_RE keyword list is only a fast + # path used when the model did not already pick an AI content op. + if not _plan_uses_ai_content(steps): + return _generative_steps(kind, prompt) + return steps + + +def _image_provider_available(owner: Optional[str]) -> bool: + try: + from src.ai_interaction import image_editing_provider_available + + return bool(image_editing_provider_available(owner)) + except Exception: + return True # unknown: let the tool attempt and report its own error + + +def _state_dims(state: Any) -> Optional[Tuple[int, int]]: + """Current canvas size from the client-sent state (flat or nested).""" + if not isinstance(state, dict): + return None + candidates = [state] + for key in ("document", "document_info", "info"): + if isinstance(state.get(key), dict): + candidates.append(state[key]) + for src in candidates: + w = src.get("width") or src.get("imgWidth") + h = src.get("height") or src.get("imgHeight") + try: + if w and h: + return (int(round(float(w))), int(round(float(h)))) + except Exception: + continue + return None + + +def _title_text(prompt: str) -> Optional[str]: + quoted = re.search(r"[\"'\u201c\u201d\u2018\u2019]([^\"'\u201c\u201d\u2018\u2019]{1,40})[\"'\u201c\u201d\u2018\u2019]", prompt) + if quoted and quoted.group(1).strip(): + return quoted.group(1).strip() + caps = re.search(r"\b([A-Z][A-Z0-9]{1,20})\b", prompt) + if caps and caps.group(1).lower() not in {"add", "title", "text", "psd"}: + return caps.group(1) + return None + + +def _crop_rect_for_aspect(cur: Tuple[int, int], target: Tuple[int, int]) -> Optional[Dict[str, int]]: + cw, ch = cur + tw, th = target + if cw <= 0 or ch <= 0 or tw <= 0 or th <= 0: + return None + target_ratio = tw / th + cur_ratio = cw / ch + if abs(cur_ratio - target_ratio) < 0.02: + return None # already the right aspect + if cur_ratio > target_ratio: + # too wide -> trim the sides + new_w = int(round(ch * target_ratio)) + new_w = min(cw, max(1, new_w)) + return {"x": (cw - new_w) // 2, "y": 0, "width": new_w, "height": ch} + # too tall -> trim top/bottom + new_h = int(round(cw / target_ratio)) + new_h = min(ch, max(1, new_h)) + return {"x": 0, "y": (ch - new_h) // 2, "width": cw, "height": new_h} + + +def _extend_dims(cur: Tuple[int, int], target: Tuple[int, int]) -> Tuple[int, int]: + """Target canvas that grows (never crops) toward the target aspect.""" + cw, ch = cur + target_ratio = target[0] / target[1] + cur_ratio = cw / ch + if abs(cur_ratio - target_ratio) < 0.02: + return (cw, ch) + if cur_ratio < target_ratio: + return (int(round(ch * target_ratio)), ch) # add width + return (cw, int(round(cw / target_ratio))) # add height + + +def _plan_text_steps(prompt: str) -> List[Dict[str, Any]]: + title = _title_text(prompt) or "Title" + steps: List[Dict[str, Any]] = [{ + "op": "text.add", + "params": {"text": title, "x": 40, "y": 40, "color": "#ffffff", "size": 64}, + }] + if _STYLE_RE.search(prompt): + steps.append({"op": "layers.addStyle", "params": {"type": "drop-shadow", "params": {}}}) + return steps + + +def _plan_format_steps(prompt: str, state: Any, provider_available: bool) -> Optional[List[Dict[str, Any]]]: + target = _format_target(prompt) + if not target: + return None + cur = _state_dims(state) + if not cur: + # No canvas size known: ask the client to crop to the aspect itself. + return None + steps: List[Dict[str, Any]] = [] + if _extend_requested(prompt): + new_w, new_h = _extend_dims(cur, target) + if (new_w, new_h) == cur: + return None + if provider_available: + steps.append({"op": "ai.generativeExpand", "params": { + "width": new_w, "height": new_h, + "anchor": "center", "mode": "instruction", "feather": True, + "steps": 8, + "prompt": "extend the surrounding scene and sky naturally into the new border, " + "keep the original photo unchanged"}}) + else: + steps.append({"op": "document.resizeCanvas", "params": { + "width": new_w, "height": new_h, "anchor": "center"}}) + else: + rect = _crop_rect_for_aspect(cur, target) + if rect: + steps.append({"op": "document.crop", "params": rect}) + # Only resize when an exact output size was requested. + if abs(target[0] / target[1] - cur[0] / cur[1]) >= 0.02 or target in ((1080, 1350), (1080, 1080), (1080, 1920), (1920, 1080)): + steps.append({"op": "document.resizeCanvas", "params": { + "width": target[0], "height": target[1], "anchor": "center"}}) + if not steps: + return None + # Optional caption/title that came with the format request. + if _is_text_request(prompt): + steps.extend(_plan_text_steps(prompt)) + return steps + + +def _plan_psd_steps(prompt: str) -> Optional[List[Dict[str, Any]]]: + if _PSD_RE.search(prompt): + return [{"op": "document.savePsd", "params": {}}] + return None + + +def _plan_special( + prompt: str, state: Any, provider_available: bool, +) -> Optional[Tuple[List[Dict[str, Any]], str, str]]: + """Deterministic overrides for intents the model reliably botches. + + Canvas format/crop/extend, text + layer styles and PSD export must not be + left to the model. Returns (steps, reply, note) or None. These never need an + image provider except generative border fill, used only when one exists. + """ + # Targeted recolour is fully deterministic (object mask + masked hue shift) + # and must never be left to the model, which sometimes substitutes a + # generative placeImage. Highest priority so it can never vary. + recolor = style_guide.recolor_plan(prompt) + if recolor: + return list(recolor["steps"]), str(recolor["reply"]), "" + if _format_target(prompt): + steps = _plan_format_steps(prompt, state, provider_available) + if not steps: + return None + target = _format_target(prompt) + reply = f"Setting the canvas to {target[0]}\u00d7{target[1]}." + note = "" + if _extend_requested(prompt) and not provider_available: + note = ("No image generation provider is configured, so the new border " + "was extended with edge/transparent pixels.") + return steps, reply, note + if _PSD_RE.search(prompt): + steps = _plan_psd_steps(prompt) + return (steps, "Saving a layered PSD.", "") if steps else None + if _is_text_request(prompt): + steps = _plan_text_steps(prompt) + if steps: + reply = ("Adding the title with a drop shadow." + if _STYLE_RE.search(prompt) else "Adding the title text.") + return steps, reply, "" + # Structural ops (fill, text edit, align, rotate/flip, copy/paste, flatten, + # merge-down, colour overlay) — the guide's deterministic routing so the + # model never substitutes a generic adjustment/transform for them. + structural = style_guide.structural_plan(prompt) + if structural: + return list(structural["steps"]), str(structural["reply"]), "" + return None + + +_DIMS_CLAIM_RE = re.compile(r"(\d{2,5})\s*[x\u00d7]\s*(\d{2,5})") +_CANVAS_CHANGE_OPS = frozenset({ + "document.crop", "document.resizeCanvas", "ai.generativeExpand", "document.importImage", +}) +_TEXT_OPS = frozenset({"text.add", "layers.addStyle", "document.savePsd"}) + + +def _step_op_signature(step: Any) -> str: + """Identify an executed step by op plus its adjustment type / filter id.""" + if not isinstance(step, dict): + return "" + op = str(step.get("op") or "") + params = step.get("params") or {} + if op in ("adjustments.apply", "adjustments.applyMasked"): + return f"{op}:{params.get('type') or ''}" + if op in ("filters.apply", "filters.applyMasked"): + return f"{op}:{params.get('id') or ''}" + return op + + +# Phrases that name a specific edit. If the reply claims one of these but the +# executed steps do not contain it, the reply is rewritten from the real steps. +_OP_CLAIM_KEYWORDS: List[Tuple[str, Tuple[str, ...]]] = [ + ("photo.autoTone", ("auto tone", "auto-tone", "autotone", "auto-tune")), + ("adjustments.apply:shadows-highlights", ("shadow", "highlight")), + ("adjustments.apply:white-balance", ("white balance", "white-balance", + "temperature", "colour cast", "color cast")), + ("adjustments.apply:vibrance", ("vibrance",)), + ("adjustments.apply:brightness-contrast", ("brightness", "contrast")), + ("adjustments.apply:black-white", ("black and white", "black & white", + "grayscale", "greyscale", "monochrome")), + ("filters.apply:clarity", ("clarity", "local contrast")), + ("filters.apply:vignette", ("vignette",)), + ("filters.apply:sharpen", ("sharpen", "crisper")), + ("retouch.skin", ("skin", "complexion")), + ("retouch.teeth", ("teeth", "smile")), +] + +_OP_LABELS: Dict[str, str] = { + "photo.autoTone": "auto tone", + "adjustments.apply:shadows-highlights": "a shadow lift (highlights protected)", + "adjustments.apply:white-balance": "white balance", + "adjustments.apply:vibrance": "vibrance", + "adjustments.apply:brightness-contrast": "brightness/contrast", + "adjustments.apply:black-white": "black and white", + "filters.apply:clarity": "local contrast (clarity)", + "filters.apply:vignette": "a light vignette", + "filters.apply:sharpen": "sharpening", + "selection.face": "a face selection", + "selection.background": "a background selection", + "retouch.skin": "gentle skin retouching", + "retouch.teeth": "teeth whitening", +} + + +def _reply_from_ops(steps: List[Dict[str, Any]]) -> str: + """Describe what was actually executed, never what was planned.""" + labels: List[str] = [] + for step in steps or []: + sig = _step_op_signature(step) + label = _OP_LABELS.get(sig) + if label is None: + label = sig.split(":")[0] or "an edit" + if label not in labels: + labels.append(label) + if not labels: + return "Done." + return "Done \u2014 I applied " + ", ".join(labels) + "." + + +def _reply_matches_ops(planned_reply: str, steps: List[Dict[str, Any]]) -> bool: + """True when the reply does not claim an edit that was not executed.""" + text = (planned_reply or "").lower() + if not text: + return True + # "drop shadow" is a layer style, not a shadows/highlights adjustment. + text = text.replace("drop shadow", "drop-shadow-style") + executed = {_step_op_signature(step) for step in (steps or [])} + for sig, keywords in _OP_CLAIM_KEYWORDS: + if sig in executed: + continue + if any(keyword in text for keyword in keywords): + return False + return True + + + +def _verified_reply( + prompt: str, + steps: List[Dict[str, Any]], + results: List[Dict[str, Any]], + final_state: Any, + planned_reply: str, +) -> str: + """Build the final reply from ACTUAL executed results + final document state. + + Never let a plan's optimistic prose claim an edit that did not happen. + """ + failed = [ + r for r in (results or []) + if isinstance(r, dict) and r.get("ok") is False + ] + if failed: + op = str(failed[0].get("op") or "step") + err = failed[0].get("error") or "it did not complete" + return (f"I couldn't finish the {op} step ({err}). " + f"Nothing else was left changed — the image is back as it was.") + dims = _state_dims(final_state) + step_ops = [str(s.get("op") or "") for s in (steps or []) if isinstance(s, dict)] + claim = _DIMS_CLAIM_RE.search(planned_reply or "") + if claim and dims: + try: + claimed = (int(claim.group(1)), int(claim.group(2))) + except Exception: + claimed = None + if claimed and (abs(dims[0] - claimed[0]) > 2 or abs(dims[1] - claimed[1]) > 2): + if not any(op in _CANVAS_CHANGE_OPS for op in step_ops): + return (f"I planned to make it {claimed[0]}\u00d7{claimed[1]}, but the canvas is " + f"still {dims[0]}\u00d7{dims[1]} — the resize didn't apply. " + f"Please try again in the editor.") + if not steps and not results: + return planned_reply or "I didn't have a concrete edit to apply." + if not _is_text_request(prompt) and not _reply_matches_ops(planned_reply, steps): + return _reply_from_ops(steps) + if _is_text_request(prompt) and any(op in _TEXT_OPS for op in step_ops) and not failed: + return planned_reply or "Added the text." + if dims and any(op in _CANVAS_CHANGE_OPS for op in step_ops): + return (planned_reply or f"Done — the canvas is now {dims[0]}\u00d7{dims[1]}.") + return planned_reply + + +def set_editor_planner( + fn: Optional[Callable[[List[dict], Optional[str]], Any]], +) -> None: + global _EDITOR_PLANNER + _EDITOR_PLANNER = fn + + +_ROUTING_GUIDANCE = ( + "Routing rules (follow strictly): removing objects/people, adding new content " + "(e.g. a moon, an animal, a sign), or replacing/changing what is in the picture " + "are GENERATIVE edits -> use op \"ai_edit_image\" (op \"inpaint\" to remove, " + "\"generative\" to replace, with a descriptive prompt) or op \"generate_image\" " + "({prompt}) for added content. NEVER approximate these with selection.rectangle, " + "adjustments.apply, filters.apply, transform.* or any other deterministic op. " + "Use deterministic ops only for tone/colour/filter/crop/resize/text/selection " + "edits that do not invent or delete picture content. \"remove the background\" " + "is op \"ai_edit_image\" op \"remove-bg\". If a generative edit is requested and " + "no image provider is configured, do NOT substitute a deterministic edit: return " + "{\"reply\": \"" + _PROVIDER_REQUIRED_MESSAGE + "\", \"steps\": []}." +) + +# Strict ops vocabulary for the model (the schema is also sent as a structured +# output constraint, so the model can only emit these ops). +_SCHEMA_NOTE = ( + "Reply with ONLY a JSON object matching the ops schema. Valid step ops are the " + "deterministic editor ops above plus: \"generate_edit\" (a model image edit; " + "params {\"prompt\": str, \"mode\": \"instruction\"|\"masked\", \"mask\": optional " + "object description}) for any edit that invents, removes, replaces, recolours or " + "restyles image content; \"select_object\" (params {\"text\": object description}) " + "to select an object by description; and \"ask_user\" (params {\"question\": str, " + "\"options\": [2-3 short choices]}) when the request is ambiguous. For " + "\"generate_edit\" set params {\"prompt\": str, \"mode\": \"instruction\"|\"masked\"} " + "and do NOT set an inner \"op\" field: the server maps instruction/restyle edits to " + "the image model's \"generative\" op and masked region edits to \"inpaint\". Prefer " + "generate_edit over deterministic approximations for content changes, and ask_user " + "with 2-3 concrete choices rather than guessing a tone tweak." +) + +_SYSTEM_PROMPT = ( + "You are a photo-editor planning agent. Convert the user's request into a JSON plan of " + "editor operations. Return ONLY JSON: {\"reply\": str, \"steps\": [{\"op\": str, \"params\": {}}]}. " + "Available ops: " + ", ".join(OP_CATALOG) + ", plus ai_edit_image and generate_image. " + + _ROUTING_GUIDANCE + " " + + _CANONICAL_GUIDANCE + " " + _STYLE_GUIDANCE + " " + "Use selection.* before targeted edits when possible. Keep to <=6 steps. " + "If a screenshot is attached, use it to judge the current result. " + + _SCHEMA_NOTE +) + +_CHECK_SYSTEM_PROMPT = ( + "You are a photo-editor quality checker. A screenshot of the editor's CURRENT result is " + "attached along with the user's request and the iteration number. Decide whether the edit " + "satisfies the request. Return ONLY JSON: if it is complete, {\"done\": true, \"reply\": str}; " + "if more work is needed, {\"done\": false, \"reply\": str, \"steps\": [{\"op\": str, \"params\": {}}]}. " + "Available ops: " + ", ".join(OP_CATALOG) + ", plus ai_edit_image and generate_image. " + + _ROUTING_GUIDANCE + " " + + _CANONICAL_GUIDANCE + " " + _STYLE_GUIDANCE + " " + "Never repeat steps that already ran; <=4 steps. " + + _SCHEMA_NOTE +) + + +_GENERIC_TONE_OPS = frozenset({"photo.autoTone", "adjustments.apply", "filters.apply"}) + + +def _plan_is_generic_tone_only(steps: Any) -> bool: + """True when a plan is nothing but whole-image tone/colour/filter steps.""" + items = [s for s in (steps or []) if isinstance(s, dict)] + return bool(items) and all(s.get("op") in _GENERIC_TONE_OPS for s in items) + + +def _unsure_plan(prompt: str, plan: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Demote a bare whole-image tone plan to an ask when the prompt is unsure. + + When the planner produced ONLY generic tone ops (photo.autoTone / + adjustments.apply / filters.apply) for a prompt that matches no concrete + intent and names no concrete defect, the editor is guessing ("it just adds + vibrance and brightness"). Ask the user with 2-3 concrete choices instead. + """ + if not isinstance(plan, dict): + return plan + steps = plan.get("steps") or [] + if not _plan_is_generic_tone_only(steps): + return plan + if _generative_kind(prompt): + return plan + if style_guide.has_concrete_intent(prompt): + return plan + return style_guide.ask_user_plan() + + +def _fallback_plan(prompt: str) -> Dict[str, Any]: + kind = _generative_kind(prompt) + if kind: + label = {"remove": "object removal", "add": "content addition", + "replace": "content replacement", "restyle": "artistic restyle"}[kind] + return { + "reply": f"Planned an AI {label}.", + "planner": "fallback", + "steps": _generative_steps(kind, prompt), + } + guided = style_guide.fallback_plan(prompt) + if guided is not None: + guided["steps"] = style_guide.enforce_guards(guided.get("steps") or []) + return guided + # Nothing mapped to a concrete intent: ASK instead of silently applying a + # default tone tweak. The old fallback emitted a generic + # photo.autoTone/adjustments/filters stack here, which is the "it just adds + # vibrance and brightness" bug for unrecognized prompts. + return style_guide.ask_user_plan() + + +def _parse_plan(raw: Any) -> Optional[Dict[str, Any]]: + if isinstance(raw, dict): + data = raw + else: + text = raw if isinstance(raw, str) else "" + if not text: + return None + try: + data = json.loads(text) + except Exception: + start, end = text.find("{"), text.rfind("}") + if start < 0 or end <= start: + return None + try: + data = json.loads(text[start:end + 1]) + except Exception: + return None + # Every plan is validated against the strict ops schema; an invalid or empty + # plan returns None so the caller falls back to ask_user (never a tone + # default). See _validate_plan. + return _validate_plan(data) + + +def _normalize_steps(body: Dict[str, Any]) -> List[Dict[str, Any]]: + if isinstance(body.get("steps"), list): + raw = body["steps"] + elif body.get("op"): + raw = [{"op": body["op"], "params": body.get("params") or {}}] + else: + raw = [] + out: List[Dict[str, Any]] = [] + for step in raw[:_MAX_STEPS]: + if not isinstance(step, dict): + continue + op = str(step.get("op") or "") + if op not in _OP_SET: + continue + params = step.get("params") + out.append({"op": op, "params": params if isinstance(params, dict) else {}}) + return out + + +async def _json_object(request: Request) -> Dict[str, Any]: + try: + body = await request.json() + except (ValueError, UnicodeDecodeError): + raise HTTPException(400, "Expected a JSON object") + if not isinstance(body, dict): + raise HTTPException(400, "Expected a JSON object") + return body + + +def _prompt(value: Any) -> str: + text = str(value or "").strip() + if len(text) > 4096: + raise HTTPException(413, "Editor prompt is too long") + return text + + +def _bridge_timeout(value: Any) -> float: + try: + timeout = float(value or _BRIDGE_DEFAULT_TIMEOUT) + except (TypeError, ValueError, OverflowError): + raise HTTPException(400, "Invalid bridge timeout") + if not math.isfinite(timeout) or timeout <= 0 or timeout > 900: + raise HTTPException(400, "Invalid bridge timeout") + return timeout + + +def _is_admin(request: Request) -> bool: + """Return True only for authorized administrator browser sessions or auth-disabled single-user mode. + + Delegated credentials (bearer API tokens) are explicitly rejected: + per src/tool_security.py, bearer tokens are long-lived credentials handed to + third parties/scripts and must remain capped at non-admin policies even + when minted by an administrator. + """ + # 1. Delegated credentials (bearer API tokens) must NEVER receive admin metadata privileges + if getattr(request.state, "api_token", False): + return False + + # 2. In-process internal tool bypass (loopback agent tools) + try: + from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN, INTERNAL_TOOL_USER + import secrets + hdr = request.headers.get(INTERNAL_TOOL_HEADER) + if hdr and secrets.compare_digest(hdr, INTERNAL_TOOL_TOKEN): + return True + if getattr(request.state, "current_user", None) == INTERNAL_TOOL_USER: + return True + except Exception: + pass + + # 3. Explicitly auth-disabled single-user operation (AUTH_ENABLED=false) + try: + from src.auth_helpers import _auth_disabled + if _auth_disabled(): + return True + except Exception: + pass + + # 4. Middleware require_admin + try: + from core.middleware import require_admin + require_admin(request) + return True + except Exception: + pass + + # 5. Direct session/user check when request.state.current_user is set + try: + auth_mgr = getattr(request.app.state, "auth_manager", None) + if auth_mgr and getattr(auth_mgr, "is_configured", False): + from src.auth_helpers import get_current_user + user = get_current_user(request) or getattr(request.state, "current_user", None) + if user and getattr(auth_mgr, "is_admin", None) and auth_mgr.is_admin(user): + return True + except Exception: + pass + + return False + + +def _user(request: Request) -> Optional[str]: + # Handle API bearer tokens explicitly: require 'chat' scope and attribute to real owner + if getattr(request.state, "api_token", False): + from src.auth_helpers import require_api_token_scope + return require_api_token_scope(request, "chat") + from src.auth_helpers import get_current_user, _auth_disabled + u = get_current_user(request) + if u: + return u + if _auth_disabled(): + return None + auth_mgr = getattr(request.app.state, "auth_manager", None) + if auth_mgr is not None and getattr(auth_mgr, "is_configured", False): + raise HTTPException(401, "Not authenticated") + client = getattr(request, "client", None) + host = (client.host if client else "") or "" + if host in ("127.0.0.1", "::1", "localhost", "testclient") or not host: + return None + raise HTTPException(401, "Not authenticated") + + +async def _default_editor_planner(messages: List[dict], owner: Optional[str]) -> Optional[str]: + """Plan with the local editor-agent model, falling back through the chain. + + Order: per-request ``agent_model`` override -> local Qwen3.5-9B (vision, + screenshot attached) -> the same model text-only with a VL caption of the + screenshot -> the hosted default/utility chat model. The model actually + used is recorded so the route can report it. + """ + result = await editor_agent_model.plan_with_editor_agent( + messages, owner, override=editor_agent_model.get_override(), + response_format=PLAN_RESPONSE_FORMAT, + ) + if result is None: + return None + raw, model_used, source = result + editor_agent_model.set_last_model(model_used, source) + return raw + + +def setup_editor_agent_routes() -> APIRouter: + router = APIRouter(tags=["editor-agent"]) + + if _EDITOR_PLANNER is None: + set_editor_planner(_default_editor_planner) + + # scoped session -> list of that session's live subscriber queues; id -> result future. + # A command is BROADCAST to every live subscriber (per-connection queue), never to a + # single shared queue: EventSource reconnects overlap, and with one shared queue a + # command could be consumed by a stale subscriber while the active page got nothing. + subscribers: Dict[Tuple[Optional[str], str], List["asyncio.Queue[dict]"]] = {} + backlog: Dict[Tuple[Optional[str], str], List[dict]] = {} + pending: Dict[str, Tuple[Optional[str], asyncio.Future]] = {} + max_pending = 1024 + + def _subscribe(session: str) -> "asyncio.Queue[dict]": + q: "asyncio.Queue[dict]" = asyncio.Queue(maxsize=_QUEUE_MAX) + subscribers.setdefault(session, []).append(q) + # Replay anything enqueued while this session had no live subscriber. + for cmd in backlog.pop(session, []) or []: + try: + q.put_nowait(cmd) + except asyncio.QueueFull: + break + return q + + def _unsubscribe(session: str, q: "asyncio.Queue[dict]") -> None: + subs = subscribers.get(session) + if subs and q in subs: + subs.remove(q) + if not subs: + subscribers.pop(session, None) + + def _broadcast(session: str, cmd: dict) -> int: + subs = subscribers.get(session) or [] + if subs: + for q in list(subs): + try: + q.put_nowait(cmd) + except asyncio.QueueFull: + pass + return len(subs) + # No live subscriber yet: hold it so a subscriber that connects shortly + # after still receives the command (bounded ring buffer). + bl = backlog.setdefault(session, []) + bl.append(cmd) + if len(bl) > _QUEUE_MAX: + del bl[0: len(bl) - _QUEUE_MAX] + return 0 + + def _expire_command(session, cid, fut): + entry = pending.get(cid) + if entry is not None and entry[1] is fut: + pending.pop(cid, None) + if not fut.done(): + fut.cancel() + queued = backlog.get(session) + if queued: + backlog[session] = [cmd for cmd in queued if cmd.get("id") != cid] + if not backlog[session]: + backlog.pop(session, None) + + def _scoped(session: Optional[str], owner: Optional[str]) -> Tuple[Optional[str], str]: + # Usernames and session ids may both contain colons. + return (owner, str(session or 'default')) + + def _scope_label(scope: Tuple[Optional[str], str]) -> str: + return f"{scope[0] or 'anon'}:{scope[1]}" + + @router.post("/api/editor/plan") + async def plan(request: Request) -> JSONResponse: + owner = _user(request) + body = await _json_object(request) + prompt = _prompt(body.get("prompt")) + state = body.get("state") or {} + mode = str(body.get("mode") or "plan").strip().lower() + screenshot = body.get("screenshot") + agent_model = str(body.get("agent_model") or "").strip() + if not prompt and mode != "check": + return JSONResponse({"reply": "No prompt provided.", "planner": "none", "steps": [], + "agent_model": agent_model or editor_agent_model.local_model()}) + + system = _CHECK_SYSTEM_PROMPT if mode == "check" else _SYSTEM_PROMPT + text = json.dumps({"prompt": prompt, "state": state}, ensure_ascii=False) + content: Any = text + if isinstance(screenshot, str) and screenshot.startswith("data:image/"): + content = [ + {"type": "text", "text": text}, + {"type": "image_url", "image_url": {"url": screenshot}}, + ] + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": content}, + ] + + raw: Any = None + planner = _EDITOR_PLANNER + editor_agent_model.set_override(agent_model) + editor_agent_model.set_last_model("", "") + if planner is not None: + try: + raw = planner(messages, owner) + if inspect.isawaitable(raw): + raw = await raw + except Exception: + raw = None + _used = editor_agent_model.get_last_model() + parsed = _parse_plan(raw) + if _used and _used[0]: + model_used = _used[0] + elif parsed: + model_used = agent_model or editor_agent_model.local_model() + else: + model_used = "fallback" if agent_model else editor_agent_model.local_model() + if isinstance(parsed, dict): + parsed["agent_model"] = model_used + if mode != "check": + _special = _plan_special(prompt, state, _image_provider_available(owner)) + if _special: + _special_steps, _special_reply, _ = _special + return JSONResponse({ + "reply": _special_reply, + "planner": "deterministic", + "steps": style_guide.enforce_guards(_special_steps) or _special_steps, + "agent_model": model_used, + }) + if parsed and mode != "check": + _strengthened = style_guide.strengthen(prompt, list(parsed.get("steps") or [])) + if _strengthened != parsed.get("steps"): + parsed["steps"] = _strengthened + parsed["planner"] = str(parsed.get("planner") or "llm") + "+guide" + # Keep /plan and /agent consistent: a generative intent must never be + # left on a deterministic substitute (e.g. a model returning only + # selection/adjustment steps for 'remove the person on the left'). + _generative = _ensure_generative(prompt, list(parsed.get("steps") or [])) + if _generative != parsed.get("steps"): + parsed["steps"] = _generative + parsed["planner"] = str(parsed.get("planner") or "llm") + "+generative" + # Same safety guards the /agent path applies, so /plan never returns + # a stacked-lift or a global adjustment after a masked recolour. + parsed["steps"] = style_guide.enforce_guards(list(parsed.get("steps") or [])) + # A bare whole-image tone plan for an unspecified prompt is a guess: + # ask the user instead of silently tweaking. + _maybe = _unsure_plan(prompt, parsed) + if _maybe is not parsed and isinstance(_maybe, dict): + _maybe["agent_model"] = model_used + parsed = _maybe + + if mode == "check": + # Never let a self-check re-plan substitute a generative placeImage + # for a deterministic recolour; re-issue the masked-hue plan instead. + _recolor = style_guide.recolor_plan(prompt) + if _recolor and _plan_uses_ai_content(list((parsed or {}).get("steps") or [])): + return JSONResponse({ + "reply": str(_recolor["reply"]), + "planner": "deterministic", + "steps": style_guide.enforce_guards(list(_recolor["steps"])), + "agent_model": model_used, + }) + if parsed and (parsed.get("done") or parsed.get("steps")): + return JSONResponse(parsed) + return JSONResponse({ + "done": True, + "reply": (parsed or {}).get("reply") or "The edit looks complete.", + "planner": (parsed or {}).get("planner") or "check", + "steps": [], + "agent_model": model_used, + }) + if parsed: + return JSONResponse(parsed) + _fallback = _fallback_plan(prompt) + _fallback["agent_model"] = model_used + return JSONResponse(_fallback) + + @router.post("/api/editor/bridge/command") + async def bridge_command(request: Request) -> JSONResponse: + owner = _user(request) + body = await _json_object(request) + session = _scoped(body.get("session"), owner) + steps = _normalize_steps(body) + wait = bool(body.get("wait")) + timeout = _bridge_timeout(body.get("timeout")) + ids: List[str] = [] + if len(pending) + len(steps) > max_pending: + raise HTTPException(429, "Too many pending editor commands") + for step in steps: + cid = uuid.uuid4().hex + ids.append(cid) + fut = asyncio.get_event_loop().create_future() + pending[cid] = (owner, fut) + _broadcast(session, {"id": cid, "op": step["op"], "params": step["params"]}) + + if wait and len(steps) == 1: + cid = ids[0] + entry = pending.get(cid) + fut = entry[1] + try: + res = await asyncio.wait_for(fut, timeout=timeout) + if isinstance(res, dict): + res.setdefault("status", "completed" if res.get("ok") else "failed") + res.setdefault("completed", bool(res.get("ok"))) + return JSONResponse(res) + return JSONResponse({"id": cid, "ok": True, "status": "completed", "completed": True, "result": res}) + except asyncio.TimeoutError: + return JSONResponse({"ok": False, "error": "timeout", "status": "timed_out", "completed": False, "id": cid}, status_code=504) + finally: + _expire_command(session, cid, fut) + else: + for cid in ids: + entry = pending.get(cid) + if entry is not None: + asyncio.get_running_loop().call_later(timeout, _expire_command, session, cid, entry[1]) + + return JSONResponse({ + "queued": ids, + "session": _scope_label(session), + "status": "queued", + "completed": False, + "ok": False, + }) + + @router.get("/api/editor/bridge/events") + async def bridge_events(request: Request, session: str = "default") -> StreamingResponse: + owner = _user(request) + scoped = _scoped(session, owner) + + async def gen(): + q = _subscribe(scoped) + try: + yield ": connected\n\n" + while not _shutdown.is_shutting_down(): + try: + cmd = await asyncio.wait_for(q.get(), timeout=15) + yield f"data: {json.dumps(cmd)}\n\n" + except asyncio.TimeoutError: + yield ": keepalive\n\n" + finally: + _unsubscribe(scoped, q) + + return StreamingResponse(gen(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + @router.post("/api/editor/bridge/result") + async def bridge_result(request: Request) -> JSONResponse: + owner = _user(request) + body = await _json_object(request) + cid = str(body.get("id") or "") + entry = pending.get(cid) + if entry is None: + return JSONResponse({"ok": True}) + if entry[0] != owner: + return JSONResponse({"ok": False, "error": "unknown command"}, status_code=404) + _, fut = entry + if fut is not None and not fut.done(): + is_ok = bool(body.get("ok")) + fut.set_result({ + "id": cid, + "ok": is_ok, + "status": "completed" if is_ok else "failed", + "completed": is_ok, + "result": body.get("result"), + "error": body.get("error"), + "screenshot": body.get("screenshot"), + }) + return JSONResponse({"ok": True}) + + @router.post("/api/editor/bridge/await") + async def bridge_await(request: Request) -> JSONResponse: + owner = _user(request) + body = await _json_object(request) + cid = str(body.get("id") or "") + timeout = _bridge_timeout(body.get("timeout")) + entry = pending.get(cid) + if entry is None or entry[0] != owner: + return JSONResponse({"ok": False, "error": "unknown command", "status": "unknown", "completed": False}, status_code=404) + fut = entry[1] + try: + result = await asyncio.wait_for(asyncio.shield(fut), timeout=timeout) + if pending.get(cid) == (owner, fut): + pending.pop(cid, None) + res = dict(result) if isinstance(result, dict) else {"result": result, "ok": True} + res.setdefault("status", "completed" if res.get("ok") else "failed") + res["completed"] = bool(res.get("ok")) + return JSONResponse(res) + except asyncio.TimeoutError: + return JSONResponse({"ok": False, "error": "timeout", "status": "timed_out", "completed": False, "id": cid}) + except asyncio.CancelledError: + return JSONResponse({"ok": False, "error": "cancelled", "status": "cancelled", "completed": False, "id": cid}) + + _AGENT_SYSTEM_PROMPT = ( + "You are the photo editor agent. You drive the open image editor and " + "may also run AI image operations. Available editor ops: " + + ", ".join(OP_CATALOG) + + ", plus ai_edit_image and generate_image. " + "Respond with ONLY a JSON object: {\"reply\": , " + "\"steps\": [{\"op\": , \"params\": {...}}]}. " + + _ROUTING_GUIDANCE + " " + + _CANONICAL_GUIDANCE + " " + _STYLE_GUIDANCE + " " + "Prefer AI content ops for generative requests: \"remove the person on the left\" " + "-> op \"ai_edit_image\" op \"inpaint\" with a prompt; \"add a full moon in the sky\" " + "-> op \"generate_image\" with a prompt. " + "Use deterministic editor ops (adjustments.apply, filters.apply, selection.*, " + "layers.placeImage, text.add, ...) only for non-generative, precise edits. " + "Keep steps minimal and ordered. " + + _SCHEMA_NOTE + ) + + def _scoped_editor_tools() -> List[str]: + """The exact tool list offered to the model for a photo_editor turn.""" + fallback = [ + "plan_edit", "run_editor_command", "get_editor_screenshot", + "ai_edit_image", "generate_image", "ask_user", + ] + try: + from src.tool_policy import ToolPolicy + from src.tool_schemas import FUNCTION_TOOL_SCHEMAS + from src.turn_contract import resolve_turn_contract + + contract = resolve_turn_contract( + capabilities={"photo_editor"}, + schemas=FUNCTION_TOOL_SCHEMAS, + policy=ToolPolicy(), + required_capabilities={"photo_editor"}, + ) + return sorted(contract.offered) or fallback + except Exception: + return fallback + + @router.post("/api/editor/agent") + async def editor_agent(request: Request) -> JSONResponse: + """Scoped photo_editor agent turn (Quick Edit). + + Offers ONLY the editor tool family plus ask_user, plans edits, and + drives the open editor through the owner-scoped bridge. AI edit and + generation actions land as new layers; the original is never + overwritten. + """ + owner = _user(request) + body = await _json_object(request) + from src.settings import get_setting + disabled = get_setting("disabled_tools", []) or [] + if not isinstance(disabled, (list, tuple, set)): + disabled = [] + if "plan_edit" in disabled: + raise HTTPException(403, "Editor tool is disabled") + prompt = _prompt(body.get("prompt")) + session = str(body.get("editor_session") or body.get("session") or "default") + state = body.get("state") if isinstance(body.get("state"), dict) else {} + screenshot = body.get("screenshot") + # Optional user-picked image model (from the editor model selector). + model_choice = str(body.get("model") or body.get("image_model") or "").strip() + # Optional user-picked PLANNER model (the editor agent/planner model). + agent_model = str(body.get("agent_model") or "").strip() + tools_offered = [tool for tool in _scoped_editor_tools() if tool not in disabled] + scoped = _scoped(session, owner) + trace: List[Dict[str, Any]] = [ + {"type": "session", "session": _scope_label(scoped), "tools_offered": tools_offered}, + ] + if not prompt: + return JSONResponse({ + "ok": False, "reply": "No prompt provided.", + "tools_offered": tools_offered, "steps": [], "trace": trace, + "agent_model": agent_model or editor_agent_model.local_model(), + }) + text = json.dumps({"prompt": prompt, "state": state}, ensure_ascii=False) + content: Any = text + if isinstance(screenshot, str) and screenshot.startswith("data:image/"): + content = [ + {"type": "text", "text": text}, + {"type": "image_url", "image_url": {"url": screenshot}}, + ] + messages = [ + {"role": "system", "content": _AGENT_SYSTEM_PROMPT}, + {"role": "user", "content": content}, + ] + raw = None + planner = _EDITOR_PLANNER + editor_agent_model.set_override(agent_model) + editor_agent_model.set_last_model("", "") + if planner is not None: + try: + raw = planner(messages, owner) + if hasattr(raw, "__await__"): + raw = await raw + except Exception: + raw = None + _used = editor_agent_model.get_last_model() + parsed = _parse_plan(raw) + if _used and _used[0]: + model_used = _used[0] + elif parsed: + model_used = agent_model or editor_agent_model.local_model() + else: + model_used = "fallback" if agent_model else editor_agent_model.local_model() + plan = parsed or _fallback_plan(prompt) + _strengthened = style_guide.strengthen(prompt, list(plan.get("steps") or [])) + if _strengthened != plan.get("steps"): + plan = {**plan, "steps": _strengthened, + "planner": f"{plan.get('planner') or 'llm'}+guide"} + kind = _generative_kind(prompt) + provider_available = _image_provider_available(owner) + # Deterministic overrides for intents the model reliably botches: canvas + # format/crop/extend, text + layer styles, and PSD export. + _special = _plan_special(prompt, state, provider_available) + special_note = "" + if _special: + special_steps, special_reply, special_note = _special + steps = style_guide.enforce_guards(special_steps) or list(special_steps) + reply = special_reply + kind = None + plan = {**plan, "steps": steps, + "planner": f"{plan.get('planner') or 'llm'}+deterministic"} + else: + steps = _ensure_generative(prompt, list(plan.get("steps") or [])) + steps = style_guide.enforce_guards(steps) + if steps != (plan.get("steps") or []): + plan = {**plan, "steps": steps, + "planner": f"{plan.get('planner') or 'llm'}+generative"} + reply = str(plan.get("reply") or "") + # A bare whole-image tone plan for an unspecified prompt is a guess: + # ask the user instead of silently tweaking. + # A bare whole-image tone plan for an unspecified prompt is a guess: + # ask the user instead of silently tweaking. Only replace when the + # guard actually changed the plan (identity), so a model-supplied + # ask_user never wipes out a concrete generative edit. + _base_plan = {**plan, "steps": steps} + _maybe = _unsure_plan(prompt, _base_plan) + if _maybe is not _base_plan and isinstance(_maybe, dict) and _maybe.get("ask_user"): + plan = _maybe + steps = [] + reply = str(_maybe.get("reply") or "") + trace.append({ + "type": "plan", "tool": "plan_edit", + "planner": plan.get("planner"), "steps": steps, + "image_provider": provider_available, "agent_model": model_used, + }) + # A vague prompt with no concrete defect asks the user instead of + # guessing a token edit (2-3 concrete options). + ask_user_payload = plan.get("ask_user") + if isinstance(ask_user_payload, dict) and not steps and not _special: + trace.append({"type": "ask_user", + "question": ask_user_payload.get("question")}) + return JSONResponse({ + "ok": True, + "ask_user": ask_user_payload, + "ask_user_pending": True, + "reply": reply or str(ask_user_payload.get("question") or ""), + "tools_offered": tools_offered, + "steps": [], + "trace": trace, + "results": [], + "agent_model": model_used, + }) + # This route dispatches tools directly, outside execute_tool_block. + # Apply the same administrator tool switches before any step runs. + needed = {"plan_edit"} + for step in steps: + op = step.get("op") + needed.add(op if op in {"ai_edit_image", "generate_image"} else "run_editor_command") + if needed.intersection(disabled): + raise HTTPException(403, "Editor tool is disabled") + # No image provider: stop before any edit runs and tell the user plainly + # (zero canvas changes; nothing to roll back). + if _plan_uses_ai_content(steps): + from src.auth_helpers import require_privilege + require_privilege(request, "can_generate_images") + if (kind or _plan_uses_ai_content(steps)) and not provider_available: + trace.append({"type": "blocked", "reason": "image_provider_unavailable"}) + return JSONResponse({ + "ok": False, + "blocked": True, + "needs_provider": True, + "reply": _PROVIDER_REQUIRED_MESSAGE, + "message": _PROVIDER_REQUIRED_MESSAGE, + "image_provider": False, + "tools_offered": tools_offered, + "steps": steps, + "trace": trace, + "results": [], + "agent_model": model_used, + }) + needed = {"plan_edit"} + for step in steps: + op = step.get("op") + needed.add(op if op in {"ai_edit_image", "generate_image"} else "run_editor_command") + if needed.intersection(disabled): + raise HTTPException(403, "Editor tool is disabled") + results: List[Dict[str, Any]] = [] + for step in steps: + op = step.get("op") + params = step.get("params") or {} + if op in ("ai_edit_image", "generate_image"): + try: + from src.agent_tools.editor_tools import AiEditImageTool, GenerateImageLayerTool + except Exception: + results.append({"op": op, "ok": False, "error": "editor tools unavailable"}) + break + tool_payload = dict(params) + # Planner output is untrusted: it cannot redirect this turn to + # another editor session or override an explicit model choice. + tool_payload["session"] = session + # Route to the user-picked model when one is selected. + if model_choice: + tool_payload["model"] = model_choice + # Hand the AI step the canvas image the client already captured + # for this request. Without it the tool falls back to a separate + # bridge screenshot, which is not reliably answered right after a + # selection step (the masked path was losing the canvas image). + if isinstance(screenshot, str) and screenshot.startswith("data:image/"): + tool_payload["image"] = screenshot + else: + tool_payload.pop("image", None) + tool_cls = AiEditImageTool if op == "ai_edit_image" else GenerateImageLayerTool + try: + _desc, res = await tool_cls().execute( + json.dumps(tool_payload), + {"owner": owner, "session_id": _scope_label(scoped), "editor_session": session, + "incognito": body.get("incognito") is True}, + ) + except Exception: + logger.exception("Editor image operation failed") + res = {"error": "Editor image operation failed", "exit_code": 1} + res = res if isinstance(res, dict) else {} + ok = res.get("exit_code", 0) == 0 + step_result = { + k: res.get(k) for k in ("image_id", "image_url", "placed", "output", "model") + if k in res + } + results.append({ + "op": op, "ok": ok, "result": step_result, + # Surface the real provider/tool failure so the reply can + # say what went wrong instead of a bare ok:true/ok:false. + "error": None if ok else (res.get("error") or res.get("output") or "it did not complete"), + }) + trace.append({"type": "step", "op": op, "ok": ok, "result": step_result}) + if not ok: + break + continue + cid = uuid.uuid4().hex + fut = asyncio.get_event_loop().create_future() + pending[cid] = (owner, fut) + _broadcast(scoped, {"id": cid, "op": op, "params": params}) + try: + res = await asyncio.wait_for(pending[cid][1], timeout=_BRIDGE_DEFAULT_TIMEOUT) + except asyncio.TimeoutError: + results.append({"op": op, "ok": False, "error": "timeout"}) + trace.append({"type": "step", "op": op, "ok": False, "error": "timeout"}) + break + finally: + _expire_command(scoped, cid, fut) + results.append(res) + trace.append({"type": "step", "op": op, "ok": res.get("ok"), "result": res.get("result")}) + if res.get("ok") is False: + break + # Post-run verification: read the ACTUAL canvas back and base the reply + # on executed results + final document.info, never the plan. + final_state: Dict[str, Any] = {} + if steps and not any(r.get("ok") is False for r in results): + try: + _info_id = uuid.uuid4().hex + _info_fut = asyncio.get_event_loop().create_future() + pending[_info_id] = (owner, _info_fut) + _broadcast(scoped, {"id": _info_id, "op": "document.info", "params": {}}) + _info = await asyncio.wait_for(pending[_info_id][1], timeout=60) + if isinstance(_info, dict) and isinstance(_info.get("result"), dict): + final_state = _info["result"] + except Exception: + final_state = {} + finally: + _expire_command(scoped, _info_id, _info_fut) + planned_reply = reply + reply = _verified_reply(prompt, steps, results, final_state, planned_reply) + if special_note and special_note.strip() not in reply: + reply = (reply + " " + special_note.strip()).strip() + return JSONResponse({ + "ok": not any(r.get("ok") is False for r in results), + "reply": reply or "Done.", + "planned_reply": planned_reply, + "image_provider": provider_available, + "final_state": final_state, + "tools_offered": tools_offered, + "tools": ["plan_edit"], + "steps": steps, + "trace": trace, + "results": results, + "agent_model": model_used, + }) + + @router.get("/api/editor/style-guide") + async def style_guide_route() -> JSONResponse: + """The shared intent -> ops guide (single source of truth). + + The browser fallback planner fetches this so the client and server + routing rules cannot drift. + """ + return JSONResponse(style_guide.as_json()) + + @router.get("/api/editor/image-models") + async def editor_image_models(request: Request) -> JSONResponse: + """List the image models available from configured, owner-visible providers. + + Each entry is ``{id, label, kind, provider}`` where ``id`` is the model id + to pass as ``model`` to ai_edit_image / generate_image, and ``kind`` is + ``edit`` (editing/inpainting models) or ``generate`` (text-to-image). + """ + owner = _user(request) + is_admin = _is_admin(request) + models: List[Dict[str, Any]] = [] + seen: set = set() + try: + from core.database import SessionLocal, ModelEndpoint + from src.auth_helpers import owner_filter + from src.endpoint_resolver import _endpoint_enabled_models + from src.image_model_ids import looks_like_image_generation_model + + db = SessionLocal() + try: + query = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), # noqa: E712 + ModelEndpoint, + owner, + ) + for ep in query.all(): + # Image-only endpoints (the local Qwen server) list their + # models verbatim. Gateway/chat endpoints (OpenRouter, …) + # mix chat and image models, so expose only ids that are + # recognisably image-capable — the same test the request + # router uses. + image_endpoint = getattr(ep, "model_type", None) == "image" + endpoint_id = getattr(ep, "id", None) + provider = getattr(ep, "name", None) + if not provider: + provider = ( + getattr(ep, "base_url", "") + if is_admin + else (f"Endpoint {endpoint_id}" if endpoint_id else "image") + ) + provider = provider or "image" + for name in _endpoint_enabled_models(ep): + if not image_endpoint and not looks_like_image_generation_model(name): + continue + key = (str(endpoint_id or provider), name) + if key in seen: + continue + seen.add(key) + entry = { + "id": name, + "label": name, + "kind": "edit" if _EDIT_MODEL_RE.search(name) else "generate", + "provider": provider, + "endpoint_id": endpoint_id, + } + if is_admin: + entry["base_url"] = getattr(ep, "base_url", "") or "" + models.append(entry) + finally: + db.close() + except Exception: + logger.warning("editor image-models lookup failed", exc_info=True) + return JSONResponse({"models": models}) + + @router.get("/api/editor/agent-models") + async def editor_agent_models(request: Request) -> JSONResponse: + """List the models the editor agent/planner can run on. + + Each entry is ``{id, label, source, vision}`` where ``id`` is + passed as ``agent_model`` to /api/editor/plan or /api/editor/agent, and + ``source`` is ``local`` (the local Qwen3.5-9B vision server) or + ``hosted`` (the configured default/utility chat model that the planner + falls back to). + For non-admin users, private URLs and internal probe status are omitted. + """ + owner = _user(request) + is_admin = _is_admin(request) + try: + models = await editor_agent_model.list_editor_agent_models(owner) + if not is_admin: + sanitized = [] + for m in models: + m_copy = dict(m) + m_copy.pop("base_url", None) + sanitized.append(m_copy) + models = sanitized + except Exception: + logger.warning("editor agent-models lookup failed", exc_info=True) + models = [] + + probe_info = {} + if is_admin: + probe_info = editor_agent_model.probe_status() + if not probe_info: + editor_agent_model.startup_probe() + probe_info = editor_agent_model.probe_status() + + return JSONResponse({ + "models": models, + "probe": probe_info, + }) + + @router.post("/api/editor/self-check") + async def editor_self_check(request: Request) -> JSONResponse: + """Same-origin VL self-check: compare BEFORE/AFTER against the intent. + + A configured vision server may not provide browser CORS headers. This route + proxies the two images + intent to the agent VLM and returns + ``{done, issues, retry_hint, text}`` (``done=True`` with + ``reason="unavailable"`` on any failure so the client never loops). + """ + owner = _user(request) + body = await _json_object(request) + result = await editor_agent_model.self_check( + body.get("before"), + body.get("after"), + _prompt(body.get("prompt") or body.get("intent")), + model=str(body.get("model") or "").strip() or None, + owner=owner, + ) + return JSONResponse(result) + + return router diff --git a/routes/email/email_routes.py b/routes/email/email_routes.py index b8329eef2..6060cd568 100644 --- a/routes/email/email_routes.py +++ b/routes/email/email_routes.py @@ -43,6 +43,7 @@ from starlette.concurrency import run_in_threadpool from fastapi import APIRouter, Query, UploadFile, File, BackgroundTasks, HTTPException, Depends, Request from fastapi.responses import FileResponse, StreamingResponse from src.constants import DATA_DIR +from src import shutdown as _shutdown from src.path_confinement import confine from src.llm_core import llm_call_async @@ -1826,20 +1827,75 @@ def setup_email_routes(): # a cached message body. return (account_id or "", folder, str(uid), owner) + _LIST_STALE = {} + _LIST_LOCK = _threading.RLock() + _LIST_CACHE_LOCK = _LIST_LOCK + # Invalidation fences reads already in flight, including background refresh. + _LIST_EPOCH = [0] + _LIST_CACHE_GENERATION = _LIST_EPOCH + def _list_cache_get(key): - v = _LIST_CACHE.get(key) - if not v: return None - if v[0] < _time.monotonic(): - _LIST_CACHE.pop(key, None) - return None - return v[1] + with _LIST_LOCK: + v = _LIST_CACHE.get(key) + if not v: return None + if v[0] < _time.monotonic(): + _LIST_CACHE.pop(key, None) + return None + return v[1] + + def _list_stale_get(key): + with _LIST_LOCK: + return _LIST_STALE.get(key) + + def _list_stale_put(key, value): + _LIST_STALE[key] = value + if len(_LIST_STALE) > 96: + for k in list(_LIST_STALE.keys())[:-48]: + _LIST_STALE.pop(k, None) + + _LIST_REFRESHING = set() def _list_cache_put(key, value): - _LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value) - # Cap size - if len(_LIST_CACHE) > 64: - for k in list(_LIST_CACHE.keys())[:-32]: - _LIST_CACHE.pop(k, None) + with _LIST_LOCK: + _LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value) + # Also snapshot the last-good response for stale-while-revalidate. + _list_stale_put(key, value) + if len(_LIST_CACHE) > 64: + for k in list(_LIST_CACHE.keys())[:-32]: + _LIST_CACHE.pop(k, None) + + def _schedule_list_refresh(sync_args, cache_key, folder, account_id, owner): + """Refresh a list snapshot off the event loop, at most once per key. + + Runs the blocking IMAP fetch in a daemon thread, then repopulates both + the TTL cache and the stale snapshot so the next list view is warm. + """ + with _LIST_LOCK: + if cache_key in _LIST_REFRESHING: + return + _LIST_REFRESHING.add(cache_key) + epoch = _LIST_EPOCH[0] + + def _run(): + try: + refreshed = _list_emails_sync(*sync_args) + if refreshed and not refreshed.get("error"): + with _LIST_LOCK: + if epoch != _LIST_EPOCH[0]: + return + _list_cache_put(cache_key, refreshed) + try: + _schedule_recent_email_warm( + refreshed.get("emails") or [], folder, account_id, owner) + except Exception: + pass + except Exception as exc: # pragma: no cover - background best effort + logger.debug("email list background refresh failed: %s", exc) + finally: + with _LIST_LOCK: + _LIST_REFRESHING.discard(cache_key) + + _threading.Thread(target=_run, daemon=True, name="email-list-refresh").start() def _folder_cache_get(account_id, owner): key = (account_id or "", owner or "") @@ -1874,37 +1930,74 @@ def setup_email_routes(): hook(account_id, folder) except Exception: pass - if account_id is None and folder is None: - _LIST_CACHE.clear() - return - for k in list(_LIST_CACHE.keys()): - k_acct = k[0] if len(k) > 0 else "" - k_folder = k[1] if len(k) > 1 else "" - if (account_id is None or k_acct == (account_id or "")) and \ - (folder is None or k_folder == folder): - _LIST_CACHE.pop(k, None) + with _LIST_LOCK: + _LIST_EPOCH[0] += 1 + if account_id is None and folder is None: + _LIST_CACHE.clear() + _LIST_STALE.clear() + return + for cache in (_LIST_CACHE, _LIST_STALE): + for k in list(cache.keys()): + k_acct = k[0] if len(k) > 0 else "" + k_folder = k[1] if len(k) > 1 else "" + if (account_id is None or k_acct == (account_id or "")) and \ + (folder is None or k_folder == folder): + cache.pop(k, None) + + def _index_update_flags(owner, account_id, folder, uid, flag, add): + with _LIST_LOCK: + _email_index_update_flags(owner, account_id, folder, uid, flag, add) + _invalidate_list_cache(account_id, folder) + + def _index_delete(owner, account_id, folder, uid): + with _LIST_LOCK: + _email_index_delete(owner, account_id, folder, uid) + _invalidate_list_cache(account_id, folder) def _update_list_cache_seen(account_id, folder, uid, seen: bool): uid_s = str(uid) - for key, (expires_at, value) in list(_LIST_CACHE.items()): - if key[0] != (account_id or "") or key[1] != folder: - continue - emails = list((value or {}).get("emails") or []) - changed = False - if seen and key[2] == "unread": - kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s] - if len(kept) != len(emails): - value = dict(value) - value["emails"] = kept - value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept))) - changed = True - else: - for e in emails: - if str((e or {}).get("uid") or "") == uid_s: - e["is_read"] = bool(seen) + with _LIST_LOCK: + for key, (expires_at, value) in list(_LIST_CACHE.items()): + if key[0] != (account_id or "") or key[1] != folder: + continue + emails = list((value or {}).get("emails") or []) + changed = False + if seen and key[2] == "unread": + kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s] + if len(kept) != len(emails): + value = dict(value) + value["emails"] = kept + value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept))) changed = True - if changed: - _LIST_CACHE[key] = (expires_at, value) + else: + for e in emails: + if str((e or {}).get("uid") or "") == uid_s: + e["is_read"] = bool(seen) + changed = True + if changed: + _LIST_CACHE[key] = (expires_at, value) + if key in _LIST_STALE: + _LIST_STALE[key] = value + + for key, value in list(_LIST_STALE.items()): + if key[0] != (account_id or "") or key[1] != folder: + continue + emails = list((value or {}).get("emails") or []) + changed = False + if seen and key[2] == "unread": + kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s] + if len(kept) != len(emails): + value = dict(value) + value["emails"] = kept + value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept))) + changed = True + else: + for e in emails: + if str((e or {}).get("uid") or "") == uid_s: + e["is_read"] = bool(seen) + changed = True + if changed: + _LIST_STALE[key] = value def _read_cache_get(key): v = _READ_CACHE.get(key) @@ -2284,6 +2377,8 @@ def setup_email_routes(): SECURITY: `owner` is propagated so when `account_id` is missing, the fallback config lookup is scoped to this user's accounts only. """ + with _LIST_CACHE_LOCK: + index_generation = _LIST_CACHE_GENERATION[0] conn = None conn_ok = False try: @@ -2578,7 +2673,9 @@ def setup_email_routes(): if parsed: fetched_emails.append(parsed) cached_rows[parsed["uid"]] = parsed - _email_index_upsert(owner, account_id, folder, fetched_emails) + with _LIST_CACHE_LOCK: + if index_generation == _LIST_CACHE_GENERATION[0]: + _email_index_upsert(owner, account_id, folder, fetched_emails) for uid_num in uid_order: row = cached_rows.get(uid_num) if not row: @@ -2867,6 +2964,48 @@ def setup_email_routes(): owner, account_id or "", folder, filter, limit, offset, elapsed_ms, ) return cached + # Stale-while-revalidate: never block a plain list view on IMAP while + # we already hold a recent snapshot or a durable index. Serve it now + # with a `stale` marker and refresh in the background so the next view + # is warm. Blocking IMAP only remains for a genuine cold start with no + # snapshot and no indexed rows (e.g. a brand-new account/folder). + if not manual_refresh and not from_addr and not has_attachments and not (date_from or date_to): + snapshot = _list_stale_get(ck) + if snapshot is None: + try: + indexed_emails, indexed_total, indexed_at = _email_index_list( + owner, account_id, folder, filter, limit, offset, False, + ) + except Exception: + indexed_emails, indexed_total, indexed_at = [], 0, None + if indexed_total: + snapshot = { + "emails": indexed_emails, + "total": indexed_total, + "folder": folder, + "offset": offset, + "sync": { + "source": "index", + "indexed": indexed_total, + "updated_at": indexed_at, + }, + } + if snapshot is not None: + payload = dict(snapshot) + payload["emails"] = list(payload.get("emails") or []) + _hide_unlinked_calendar_tags(payload["emails"]) + sync_meta = dict(payload.get("sync") or {}) + sync_meta["source"] = "cache_stale" if _list_stale_get(ck) is not None else "index" + sync_meta["stale"] = True + payload["sync"] = sync_meta + _schedule_list_refresh( + (folder, limit, offset, filter, account_id, from_addr, + bool(has_attachments), owner, True, date_from or "", date_to or ""), + ck, folder, account_id, owner, + ) + return payload + with _LIST_CACHE_LOCK: + cache_generation = _LIST_CACHE_GENERATION[0] result = await _asyncio.to_thread( _list_emails_sync, folder, limit, offset, filter, account_id, from_addr, bool(has_attachments), owner, manual_refresh, date_from or "", date_to or "", @@ -2875,7 +3014,9 @@ def setup_email_routes(): if offset == 0 and not from_addr and not has_attachments and filter in ("all", "unread", "unanswered", "undone"): _record_email_received_events(owner, account_id, folder, result.get("emails") or []) _schedule_recent_email_warm(result.get("emails") or [], folder, account_id, owner) - _list_cache_put(ck, result) + with _LIST_CACHE_LOCK: + if cache_generation == _LIST_CACHE_GENERATION[0]: + _list_cache_put(ck, result) elapsed_ms = int((_time.monotonic() - started_at) * 1000) if elapsed_ms > 1500: logger.warning( @@ -3223,7 +3364,7 @@ def setup_email_routes(): conn.select(_q(folder)) moved = _move_email_message(conn, uid, "Junk", role="junk") if moved: - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) except Exception: logger.debug("unsubscribe move-to-spam skipped", exc_info=True) @@ -3240,7 +3381,7 @@ def setup_email_routes(): if deleted: conn.expunge() if deleted: - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) except Exception as exc: delete_error = "Email was unsubscribed but could not be moved to Trash" @@ -3322,7 +3463,7 @@ def setup_email_routes(): if moved: changed += 1 cleaned_uids.append(uid) - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) else: failed += 1 except Exception: @@ -3411,6 +3552,8 @@ def setup_email_routes(): folder (Gmail does), we transparently swap to All Mail so the search surfaces archived / labelled emails too. Plain IMAP accounts fall back to whatever folder the caller specified.""" + with _LIST_CACHE_LOCK: + index_generation = _LIST_CACHE_GENERATION[0] if not q or len(q) < 2: return {"emails": [], "total": 0, "query": q} # CRLF in q would terminate the IMAP command early — reject defensively. @@ -3554,7 +3697,9 @@ def setup_email_routes(): except Exception as e: logger.warning(f"Error parsing search result {uid}: {e}") continue - _email_index_upsert(owner, account_id, effective_folder, fetched_emails) + with _LIST_CACHE_LOCK: + if index_generation == _LIST_CACHE_GENERATION[0]: + _email_index_upsert(owner, account_id, effective_folder, fetched_emails) return { "emails": emails, @@ -3691,8 +3836,7 @@ def setup_email_routes(): # accepted it, so the index and list cache cannot drift ahead of # the mailbox. if mark_seen and not mark_seen_failed: - _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) - _update_list_cache_seen(account_id, folder, uid, True) + _index_update_flags(owner, account_id, folder, uid, "\\Seen", True) related_attachments = [] if full and not _has_visible_attachments(msg): @@ -3831,8 +3975,7 @@ def setup_email_routes(): status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "(\\Seen)") if status != "OK": return False - _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) - _update_list_cache_seen(account_id, folder, uid, True) + _index_update_flags(owner, account_id, folder, uid, "\\Seen", True) return True except Exception as e: logger.warning(f"mark-seen after cached read failed uid={uid}: {e}") @@ -5064,7 +5207,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Seen", add=False): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", False) + _index_update_flags(owner, account_id, folder, uid, "\\Seen", False) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -5084,7 +5227,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Flagged", add=bool(on)): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on)) + _index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on)) _invalidate_list_cache(account_id, folder) return {"success": True, "flagged": bool(on)} except Exception as e: @@ -5102,7 +5245,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Seen", add=True): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) + _index_update_flags(owner, account_id, folder, uid, "\\Seen", True) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -5122,7 +5265,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _move_email_message(conn, uid, "Archive", role="archive"): return {"success": False, "error": "Email not found"} - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) return {"success": True} except Exception as e: @@ -5153,7 +5296,7 @@ def setup_email_routes(): # click or by another mailbox client; it is already gone # from the requested folder, so do not trap the UI on a # permanent “Email not found” error. - _email_index_delete(owner, account_id, folder, str(uid)) + _index_delete(owner, account_id, folder, str(uid)) _invalidate_list_cache(account_id, folder) return {"success": True, "already_deleted": True} trash_folder = _resolve_mail_folder(conn, "Trash", "trash") @@ -5171,9 +5314,9 @@ def setup_email_routes(): if not _move_email_message(conn, resolved_uid, trash_folder, role="trash"): logger.warning("Email delete Trash move failed uid=%s resolved_uid=%s folder=%s trash=%s", uid, resolved_uid, folder, trash_folder) return {"success": False, "error": "Could not move email to Trash"} - _email_index_delete(owner, account_id, folder, resolved_uid) + _index_delete(owner, account_id, folder, resolved_uid) if resolved_uid != str(uid): - _email_index_delete(owner, account_id, folder, str(uid)) + _index_delete(owner, account_id, folder, str(uid)) _invalidate_list_cache(account_id) return {"success": True} except Exception as e: @@ -5189,7 +5332,7 @@ def setup_email_routes(): if not _store_email_flag(conn, uid, "\\Deleted", add=True): return {"success": False, "error": "Email not found"} conn.expunge() - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -5257,7 +5400,7 @@ def setup_email_routes(): pending_expunge.append(uid) continue deleted_uids.append(uid) - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) except Exception: failed_uids.append(uid) logger.debug("Bulk email delete failed for uid=%s", uid, exc_info=True) @@ -5279,7 +5422,7 @@ def setup_email_routes(): if expunge_ok: for uid in pending_expunge: deleted_uids.append(uid) - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) else: # Prioritize preservation over cleanup: unflag \Deleted from our # copied messages so future EXPUNGE operations do not destroy them. @@ -5398,7 +5541,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _move_email_message(conn, uid, dest): return {"success": False, "error": f"Failed to move to {dest}"} - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) return {"success": True} except Exception as e: @@ -5484,7 +5627,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Answered", add=True): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Answered", True) + _index_update_flags(owner, account_id, folder, uid, "\\Answered", True) _clear_done_response_tags(owner, account_id, folder, uid) _invalidate_list_cache(account_id, folder) return {"success": True} @@ -5500,7 +5643,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Answered", add=False): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Answered", False) + _index_update_flags(owner, account_id, folder, uid, "\\Answered", False) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -6167,7 +6310,7 @@ def setup_email_routes(): try: st, _sel = imap.select(_q(_source_folder), readonly=False) if st == "OK" and _store_email_flag(imap, _source_uid, "\\Answered", add=True): - _email_index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True) + _index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True) _clear_done_response_tags(owner, _account_id, _source_folder, _source_uid) _invalidate_list_cache(_account_id, _source_folder) logger.info(f"Marked source UID {_source_uid} as \\Answered in {_source_folder}") @@ -6717,7 +6860,7 @@ def setup_email_routes(): await queue.put({'type': 'result', **result}) task = asyncio.create_task(run()) try: - while True: + while not _shutdown.is_shutting_down(): item = await queue.get() yield 'data: ' + json.dumps(item) + '\n\n' if item.get('type') == 'result': diff --git a/routes/gallery/gallery_routes.py b/routes/gallery/gallery_routes.py index 276ac90a5..690c22044 100644 --- a/routes/gallery/gallery_routes.py +++ b/routes/gallery/gallery_routes.py @@ -1,14 +1,17 @@ """Gallery routes — browsable library for photos and AI-generated images.""" import os +import asyncio import base64 +import gc import hashlib import io import logging import re +import time import uuid from pathlib import Path -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, Tuple from fastapi import APIRouter, HTTPException, Query, Request @@ -34,6 +37,241 @@ _SAM_STATE: Dict[str, Any] = {} _GROUNDING_STATE: Dict[str, Any] = {} +def _rss_mb() -> float: + """Current process RSS in MB (from /proc; -1.0 when unavailable).""" + try: + with open("/proc/self/status", "r") as fh: + for line in fh: + if line.startswith("VmRSS:"): + return round(int(line.split()[1]) / 1024.0, 1) + except Exception: + pass + return -1.0 + + +def _image_op_log_rss(op: str, phase: str) -> None: + logger.info("image-op %s %s rss_mb=%.1f", op, phase, _rss_mb()) + + +def _local_image_models_enabled() -> bool: + """Whether GB-scale local image models may be loaded IN-PROCESS. + + Default OFF: loading rembg/SAM/OWL-ViT/Real-ESRGAN in the web process + routinely OOM-kills memory-constrained hosts. Enable with + ``ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS=1`` or point ``ODYSSEUS_IMAGE_OPS_BASE`` + at a remote service that implements the same ``/api/image/*`` endpoints. + """ + return os.getenv("ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS", "").strip().lower() in ( + "1", "true", "yes", "on", + ) + + +def _remote_image_ops_base() -> str: + return (os.getenv("ODYSSEUS_IMAGE_OPS_BASE", "") or "").strip().rstrip("/") + + +def _outpaint_timeout() -> float: + """Provider timeout for the outpaint/extend proxy. + + Tall extensions can take up to ~330 s (the provider's own timeout is + 900 s), so the proxy waits 900 s by default — otherwise a long extend is + cut off at the proxy. Override with ``ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT``. + """ + try: + return float(os.getenv("ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT", "900") or 900) + except (TypeError, ValueError): + return 900.0 + + +def _unload_image_models() -> None: + """Drop lazily-loaded torch models so their RAM returns to the OS.""" + if not _SAM_STATE and not _GROUNDING_STATE: + return + _SAM_STATE.clear() + _GROUNDING_STATE.clear() + try: + import torch # type: ignore + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + gc.collect() + _image_op_log_rss("models", "unloaded") + + +async def _route_heavy_image_op(path: str, body: dict, *, label: str): + """Proxy a heavy image op to the remote ops service when configured, else + run it in-process only when explicitly enabled. + + Returns the remote JSON on success, or ``None`` to signal the caller should + run it in-process; raises HTTPException(503) with a clear message otherwise + so the app never loads a GB-scale model on a memory-constrained host. + """ + base = _remote_image_ops_base() + if base: + _image_op_log_rss(label, f"proxy->{base}") + try: + import httpx + + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post(base + path, json=body) + if resp.status_code == 200: + return resp.json() + logger.warning("image-op %s remote %s -> %s", label, path, resp.status_code) + raise HTTPException( + 503, + f"{label} is temporarily unavailable (remote service returned {resp.status_code}).", + ) + except HTTPException: + raise + except Exception as exc: + logger.warning("image-op %s remote %s failed: %s", label, path, exc) + raise HTTPException( + 503, + f"{label} is temporarily unavailable (remote service unreachable).", + ) from exc + if _local_image_models_enabled(): + return None + raise HTTPException(503, f"{label} is temporarily unavailable on this server.") + + +def _xywh_to_xyxy(bbox): + """Convert a remote image-ops [x, y, w, h] bbox to this app's [x1,y1,x2,y2].""" + try: + x, y, w, h = (float(v) for v in bbox) + return [int(x), int(y), int(x + w), int(y + h)] + except Exception: + return bbox + + +async def _remote_remove_background(body: dict): + """Proxy background removal to the remote ops service (seneca contract). + + Endpoint: POST {base}/v1/images/remove-background (multipart: image, + optional hint_mask, model, return_mask). Returns {"image": } + to match this route's local response shape, or None when the caller should + run in-process (local models explicitly enabled); raises 503 otherwise. + """ + base = _remote_image_ops_base() + if not base: + if _local_image_models_enabled(): + return None + raise HTTPException(503, "Background removal is temporarily unavailable on this server.") + img_bytes = _b64_bytes(body.get("image")) + hint_bytes = None + if body.get("hint_mask"): + hint_bytes = _b64_bytes(body.get("hint_mask"), label="hint_mask") + _image_op_log_rss("remove-bg", f"proxy->{base}") + try: + import httpx + + files = {"image": ("image.png", img_bytes, "image/png")} + if hint_bytes: + files["hint_mask"] = ("hint.png", hint_bytes, "image/png") + data = {"model": str(body.get("model") or ""), "return_mask": "true"} + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post(base + "/v1/images/remove-background", files=files, data=data) + if resp.status_code == 200: + payload = resp.json() + items = payload.get("data") or [] + if not items or not items[0].get("b64_json"): + raise HTTPException(503, "Background removal is temporarily unavailable (remote returned no image).") + out = {"image": items[0]["b64_json"]} + if items[0].get("mask_b64"): + out["mask"] = items[0]["mask_b64"] + if items[0].get("bbox"): + out["bbox"] = items[0]["bbox"] + return out + raise HTTPException( + 503, f"Background removal is temporarily unavailable (remote service returned {resp.status_code})." + ) + except HTTPException: + raise + except Exception as exc: + logger.warning("image-op remove-bg remote failed: %s", exc) + raise HTTPException(503, "Background removal is temporarily unavailable (remote service unreachable).") from exc + + +async def _remote_smart_mask(body: dict): + """Proxy SAM/grounding masking to the remote ops service (seneca contract). + + Endpoint: POST {base}/v1/images/mask (multipart: image + one of prompt/box/ + points). Returns {"mask": , "bbox": [...], "model", "device"} to + match this route's local response shape, or None when the caller should run + in-process; raises 503 otherwise. + """ + base = _remote_image_ops_base() + if not base: + if _local_image_models_enabled(): + return None + raise HTTPException(503, "Smart mask is temporarily unavailable on this server.") + img_bytes = _b64_bytes(body.get("image")) + points = body.get("points") or [] + box = body.get("box") + text = (body.get("text") or body.get("query") or "").strip() + if not points and not box and not text: + raise HTTPException(400, "Provide at least one point, box, or object text") + _image_op_log_rss("mask", f"proxy->{base}") + try: + import httpx + + data: Dict[str, Any] = {} + if text: + data["prompt"] = text + if box: + try: + x1, y1, x2, y2 = (float(v) for v in box) + data["box"] = ",".join(str(int(v)) for v in (x1, y1, x2 - x1, y2 - y1)) + except Exception as exc: + raise HTTPException(400, "Box must be [x1, y1, x2, y2]") from exc + if points: + import json as _json + + data["points"] = _json.dumps( + [[float(p["x"]), float(p["y"]), int(p.get("label", 1))] for p in points] + ) + if body.get("multimask") is not None: + data["multimask"] = str(bool(body.get("multimask"))).lower() + if body.get("dilate_px") is not None: + data["dilate_px"] = str(int(body.get("dilate_px"))) + files = {"image": ("image.png", img_bytes, "image/png")} + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post(base + "/v1/images/mask", files=files, data=data) + if resp.status_code == 200: + payload = resp.json() + masks = payload.get("masks") or [] + candidates = [m for m in masks if m.get("b64_json")] + if not candidates: + # Empty selection is a normal 200 outcome (never a 400): the + # editor shows a "nothing selected" message and changes nothing. + return { + "mask": None, + "bbox": None, + "masks": [], + "model": payload.get("model") or "sam+grounding", + "device": payload.get("device", ""), + "error": "Nothing selected for that selection.", + } + best = max(candidates, key=lambda m: float(m.get("score") or 0.0)) + return { + "mask": best["b64_json"], + "bbox": _xywh_to_xyxy(best.get("bbox")), + "label": best.get("label"), + "masks": masks, + "model": payload.get("model") or "sam+grounding", + "device": payload.get("device", ""), + } + raise HTTPException( + 503, f"Smart mask is temporarily unavailable (remote service returned {resp.status_code})." + ) + except HTTPException: + raise + except Exception as exc: + logger.warning("image-op mask remote failed: %s", exc) + raise HTTPException(503, "Smart mask is temporarily unavailable (remote service unreachable).") from exc + + def _b64_to_pil_image(image_b64: str, *, mode: str = "RGBA"): if not image_b64: raise HTTPException(400, "Missing image") @@ -50,12 +288,79 @@ def _b64_to_pil_image(image_b64: str, *, mode: str = "RGBA"): raise HTTPException(400, "Invalid image") from exc +def _b64_bytes(value: str, *, label: str = "image") -> bytes: + """Decode raw base64 OR a ``data:image/...;base64,`` URL to bytes. + + The image-edit tools post data URLs, while the browser posts raw base64, + so every decode site must accept both. + """ + if not value: + raise HTTPException(400, f"Missing {label}") + text = str(value) + if "," in text and text.split(",", 1)[0].startswith("data:"): + text = text.split(",", 1)[1] + try: + return base64.b64decode(text) + except Exception as exc: + raise HTTPException(400, f"Invalid {label}") from exc + + def _pil_image_to_b64(img, *, fmt: str = "PNG") -> str: buf = io.BytesIO() img.save(buf, format=fmt) return base64.b64encode(buf.getvalue()).decode("ascii") +def _prepare_local_edit(src_b64, mask_b64, max_side): + """Decode/normalize/cap an edit's inputs to PNG bytes at a model-friendly size. + + CPU-bound (a 4000x3000 decode+resize is ~0.2 s); run it via a thread off the + event loop. Returns (source_png_bytes, mask_png_bytes, model_w, model_h, orig_size). + """ + from PIL import Image + + img_bytes = _b64_bytes(src_b64) + source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA") + if mask_b64: + mask_bytes = _b64_bytes(mask_b64, label="mask") + mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L") + else: + mask_png = Image.new("L", source_png.size, 255) + orig_size = source_png.size + if max(orig_size) > max_side: + scale = float(max_side) / max(orig_size) + model_w = max(16, int(round(orig_size[0] * scale / 16.0)) * 16) + model_h = max(16, int(round(orig_size[1] * scale / 16.0)) * 16) + else: + model_w, model_h = orig_size + if (model_w, model_h) != orig_size: + source_work = source_png.resize((model_w, model_h), Image.LANCZOS) + mask_work = mask_png.resize((model_w, model_h), Image.LANCZOS) + else: + source_work, mask_work = source_png, mask_png + src_buf = io.BytesIO() + source_work.save(src_buf, format="PNG") + mask_buf = io.BytesIO() + mask_work.save(mask_buf, format="PNG") + return src_buf.getvalue(), mask_buf.getvalue(), model_w, model_h, orig_size + + +def _restore_local_b64(raw_b64, orig_size, model_size): + """Resize a provider result b64 back to the original dimensions (CPU-bound).""" + if tuple(model_size) == tuple(orig_size): + return raw_b64 + from PIL import Image + + try: + im = Image.open(io.BytesIO(base64.b64decode(raw_b64))).convert("RGBA") + im = im.resize(tuple(orig_size), Image.LANCZOS) + ob = io.BytesIO() + im.save(ob, format="PNG") + return base64.b64encode(ob.getvalue()).decode("ascii") + except Exception: + return raw_b64 + + def _load_sam_backend(): model_id = os.getenv("ODYSSEUS_SAM_MODEL", "facebook/sam-vit-base") cached = _SAM_STATE.get(model_id) @@ -301,7 +606,61 @@ def _first_visible_image_endpoint(db, owner: str | None): for ep in endpoints: if getattr(ep, "owner", None) == owner: return ep - return endpoints[0] if endpoints else None + if endpoints: + return endpoints[0] + # Fallback: the image model may live on a *mixed* endpoint (model_type != + # "image"), e.g. qwen-image-edit / gpt-image-1 on a shared OpenAI-compatible + # gateway or a diffusion server registered as "chat". Chat image generation + # already resolves its model by name across every enabled endpoint, so the + # editor must find the same model here instead of reporting "no endpoint". + return _first_mixed_image_endpoint(db, owner) + + +def _first_mixed_image_endpoint(db, owner: str | None): + """Best-effort image endpoint when no image-only endpoint is registered. + + Returns an enabled, owner-visible endpoint whose configured model list + includes an image-generation model, preferring the admin's configured + ``image_model`` when it is one of them. + """ + import json + from src.auth_helpers import owner_filter + from src.image_model_ids import looks_like_image_generation_model + + preferred = "" + try: + from src.settings import load_settings + preferred = str((load_settings() or {}).get("image_model") or "").strip() + except Exception: + preferred = "" + + query = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), # noqa: E712 + ModelEndpoint, + owner, + ) + fallback = None + for ep in query.all(): + if getattr(ep, "model_type", None) == "image": + continue + models = set() + for field in ("cached_models", "pinned_models"): + try: + values = getattr(ep, field, None) or [] + values = json.loads(values) if isinstance(values, str) else values + models.update(v for v in values if isinstance(v, str)) + except (TypeError, ValueError): + pass + matches = {m for m in models if looks_like_image_generation_model(m)} + if not matches: + continue + if preferred and preferred in matches: + return ep + if owner and getattr(ep, "owner", None) == owner: + return ep + if fallback is None: + fallback = ep + return fallback def _visible_image_endpoint_for_base(db, base: str, owner: str | None, model: str = ""): @@ -324,9 +683,15 @@ def _visible_image_endpoint_for_base(db, base: str, owner: str | None, model: st configured.update(v for v in values if isinstance(v, str)) except (TypeError, ValueError): pass - if model not in configured or not looks_like_image_generation_model(model): - continue - if _normalize_image_endpoint_base(getattr(ep, "base_url", "")) == target: + if model: + if model not in configured or not looks_like_image_generation_model(model): + continue + else: + if not any(looks_like_image_generation_model(v) for v in configured): + continue + ep_id = str(getattr(ep, "id", "") or "") + ep_base = _normalize_image_endpoint_base(getattr(ep, "base_url", "")) + if (ep_id and (base == ep_id or target == ep_id)) or (ep_base and ep_base == target): if owner and getattr(ep, "owner", None) == owner: return ep if fallback is None: @@ -353,13 +718,102 @@ async def _fetch_result_image_b64(url: str) -> Optional[str]: ) if not ok: raise HTTPException(502, f"Upstream returned an unsafe image URL: {reason}") - async with httpx.AsyncClient(timeout=60) as c2: + async with httpx.AsyncClient(timeout=60, follow_redirects=False) as c2: ir = await c2.get(url) if ir.status_code == 200: return base64.b64encode(ir.content).decode() return None +def _resolve_image_op_endpoint(db, owner, requested_base: str, chosen_model: str = "") -> Tuple[str, Optional[str]]: + """Resolve the provider base (with /v1) + api_key for a self-hosted image op. + + Mirrors inpaint_proxy: a client-supplied base must map to a registered, + owner-visible image endpoint (no direct use of request-body URLs). + """ + if requested_base: + ep = _visible_image_endpoint_for_base(db, requested_base, owner, chosen_model) + if not ep: + raise HTTPException(403, "Choose a registered image endpoint") + else: + ep = _first_visible_image_endpoint(db, owner) + if not ep: + raise HTTPException(400, "No image generation endpoint configured. Serve a diffusion model via Cookbook first.") + base = ep.base_url.rstrip("/") + if not base.endswith("/v1"): + base += "/v1" + return base, ep.api_key + + +async def _forward_seneca_image_op( + base: str, + api_key: Optional[str], + provider_path: str, + image_bytes: bytes, + fields: Dict[str, str], + *, + label: str, + timeout: float = 300.0, +) -> Dict[str, Any]: + """POST a single-image multipart op to the provider and return ``{image}``. + + The provider returns OpenAI-style ``{data:[{b64_json|url}]}`` or a raw + ``{image}``. ``request_id`` in ``fields`` keys the provider's progress route. + """ + import httpx + + url = base.rstrip("/") + provider_path + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + files = {"image": ("source.png", image_bytes, "image/png")} + _t0 = time.perf_counter() + try: + async with httpx.AsyncClient(timeout=timeout) as client: + r = await client.post(url, data=fields, files=files, headers=headers) + _provider_ms = (time.perf_counter() - _t0) * 1000.0 + except httpx.TimeoutException: + raise HTTPException(504, f"{label} timed out ({int(timeout)}s)") + except Exception as exc: + logger.warning("%s provider call failed: %s", label, exc) + raise HTTPException(503, f"{label} is temporarily unavailable (remote service unreachable).") + if r.status_code != 200: + logger.warning("%s provider status %s", label, r.status_code) + detail = f"{label} failed" + try: + j = r.json() + if isinstance(j, dict) and (j.get("error") or j.get("detail")): + detail = str(j.get("error") or j.get("detail"))[:200] + except Exception: + pass + raise HTTPException(r.status_code if r.status_code in (400, 404, 422) else 502, detail) + try: + result = r.json() + except ValueError as exc: + raise HTTPException(502, "Image endpoint returned invalid JSON") from exc + raw_b64: Optional[str] = None + used_model = "" + if isinstance(result, dict): + used_model = str(result.get("model") or "") + data = result.get("data") + if isinstance(data, list) and data: + item = data[0] if isinstance(data[0], dict) else {} + raw_b64 = item.get("b64_json") or None + if not raw_b64 and item.get("url"): + raw_b64 = await _fetch_result_image_b64(item["url"]) + if not raw_b64 and result.get("image"): + raw_b64 = result["image"] + if not raw_b64: + raise HTTPException(502, "Image endpoint returned no image") + logger.info( + "%s timing model_ms=%.1f total_ms=%.1f request_id=%s", + label, _provider_ms, + (time.perf_counter() - _t0) * 1000.0, fields.get("request_id") or "-", + ) + payload: Dict[str, Any] = {"image": raw_b64} + if used_model: + payload["model"] = used_model + return payload + + def setup_gallery_routes() -> APIRouter: router = APIRouter(tags=["gallery"]) @@ -602,7 +1056,7 @@ def setup_gallery_routes() -> APIRouter: # Use img2img endpoint if available, otherwise upscale via canvas on client try: - async with httpx.AsyncClient(timeout=120) as client: + async with httpx.AsyncClient(timeout=300) as client: resp = await client.post(f"{base_url}/images/upscale", json={ "image": b64, "scale": scale, }) @@ -645,7 +1099,7 @@ def setup_gallery_routes() -> APIRouter: base_url += "/v1" try: - async with httpx.AsyncClient(timeout=180) as client: + async with httpx.AsyncClient(timeout=300) as client: resp = await client.post(f"{base_url}/images/generations", json={ "prompt": prompt, "image": b64, @@ -1282,7 +1736,7 @@ def setup_gallery_routes() -> APIRouter: requested_base = (body.pop("_endpoint", "") or "").rstrip("/") # SSRF hardening: validate a client-supplied endpoint before any # outbound request (mirrors routes/embedding_routes.py). - if requested_base: + if requested_base and "://" in requested_base: from src.url_safety import check_outbound_url ok, reason = check_outbound_url( requested_base, @@ -1291,6 +1745,10 @@ def setup_gallery_routes() -> APIRouter: if not ok: raise HTTPException(400, f"Rejected endpoint URL: {reason}") chosen_model = (body.pop("_model", "") or "").strip() + # Optional client-supplied id so GET /api/image/edit-progress can poll the + # provider's step progress for this op. Ignored by providers that do not + # implement the progress contract. + progress_request_id = (body.get("request_id") or "").strip() api_key = None if not requested_base: db = SessionLocal() @@ -1338,8 +1796,10 @@ def setup_gallery_routes() -> APIRouter: raise HTTPException(500, "Pillow not installed on server") try: - img_bytes = base64.b64decode(body["image"]) - mask_bytes = base64.b64decode(body["mask"]) + img_bytes = _b64_bytes(body.get("image")) + if not body.get("mask"): + raise HTTPException(400, "A mask is required for OpenAI-compatible image edits") + mask_bytes = _b64_bytes(body.get("mask"), label="mask") source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA") mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L") # luminance # Build OpenAI mask: RGBA where alpha=255 means keep, 0 means regenerate. @@ -1390,7 +1850,11 @@ def setup_gallery_routes() -> APIRouter: } headers = {"Authorization": f"Bearer {api_key}"} try: - async with httpx.AsyncClient(timeout=120) as client: + logger.debug( + "inpaint_proxy provider edit -> model=%r size=%r base=%s", + oa_model, size, base, + ) + async with httpx.AsyncClient(timeout=300) as client: if is_openrouter: # Reference-based editing, then composite locally so # pixels outside the user's mask remain unchanged. @@ -1410,7 +1874,16 @@ def setup_gallery_routes() -> APIRouter: r = await client.post(_join_checked_gallery_endpoint(base, "/images/edits"), headers=headers, data=data, files=files) if r.status_code != 200: logger.error("inpaint_proxy OpenAI edit: status %s", r.status_code) - raise HTTPException(r.status_code, "OpenAI edit failed") + _detail = "OpenAI edit failed" + try: + _j = r.json() + if isinstance(_j, dict): + _detail = _j.get("error") or _j.get("detail") or _detail + if isinstance(_detail, dict): + _detail = _detail.get("message") or _detail + except Exception: + pass + raise HTTPException(r.status_code, f"{_detail} (provider HTTP {r.status_code})") result = r.json() raw_b64 = None if result.get("data"): @@ -1441,13 +1914,16 @@ def setup_gallery_routes() -> APIRouter: blended = Image.composite(generated, source_png, mask_png) out_buf = io.BytesIO() blended.save(out_buf, format="PNG") - return {"image": base64.b64encode(out_buf.getvalue()).decode()} + return {"image": base64.b64encode(out_buf.getvalue()).decode(), "model": oa_model} except Exception as comp_err: # Never return a full-image edit when masking fails. logger.warning(f"Inpaint compose failed: {comp_err}") raise HTTPException(502, "Could not apply the edit within the selected region") except httpx.TimeoutException: - raise HTTPException(504, "OpenAI inpaint timed out (120s)") + raise HTTPException(504, "OpenAI inpaint timed out (300s)") + except httpx.TransportError as exc: + logger.warning("inpaint_proxy: OpenAI endpoint unreachable: %s", exc) + raise HTTPException(502, f"Image server unreachable: {type(exc).__name__}") # Self-hosted diffusion server path. Newer Odysseus image # wrappers expose the OpenAI-compatible /v1/images/edits @@ -1459,45 +1935,96 @@ def setup_gallery_routes() -> APIRouter: # supports multiple models per process. Harmless if ignored. if chosen_model: body["model"] = chosen_model - async with httpx.AsyncClient(timeout=240) as client: + async with httpx.AsyncClient(timeout=300) as client: + _t0 = time.perf_counter() + _restore_ms = [0.0] try: import base64, io from PIL import Image - img_bytes = base64.b64decode(body["image"]) - mask_bytes = base64.b64decode(body["mask"]) - # Normalize both inputs to PNG bytes. Local MLX and - # Diffusers wrappers expect white mask pixels to mean - # "edit this region", which matches the editor's mask. - source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA") - mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L") - src_buf = io.BytesIO() - source_png.save(src_buf, format="PNG") - mask_buf = io.BytesIO() - mask_png.save(mask_buf, format="PNG") + # Decode/resize/encode off the event loop: a 4000x3000 input + # costs ~0.2 s of CPU that would otherwise block every other + # request (and stack under concurrent edits). + _max_side = int(os.getenv("ODYSSEUS_IMAGE_EDIT_MAX_SIDE", "768") or 768) + _src_png, _mask_png, _model_w, _model_h, _orig_size = await asyncio.to_thread( + _prepare_local_edit, body.get("image"), body.get("mask"), _max_side + ) files = { - "image": ("source.png", src_buf.getvalue(), "image/png"), - "mask": ("mask.png", mask_buf.getvalue(), "image/png"), + "image": ("source.png", _src_png, "image/png"), + "mask": ("mask.png", _mask_png, "image/png"), } data = { "model": chosen_model or body.get("model") or "", "prompt": body.get("prompt", ""), - "size": f"{int(body.get('width') or source_png.width)}x{int(body.get('height') or source_png.height)}", + "size": f"{_model_w}x{_model_h}", "n": "1", } + if progress_request_id: + data["request_id"] = progress_request_id + # Let the caller's step budget through up to a cap (quality + # edits such as removal/background-replacement/expand pass 8 + # on purpose); only fall back to a moderate default when no + # count is requested. The size cap above is what keeps the + # op responsive. + _max_steps = int(os.getenv("ODYSSEUS_IMAGE_EDIT_MAX_STEPS", "8") or 8) + if body.get("steps"): + try: + data["steps"] = str(min(max(1, int(body["steps"])), _max_steps)) + except (TypeError, ValueError): + data["steps"] = "4" + else: + data["steps"] = "4" + + async def _restore_b64(_raw: str) -> str: + """Resize a provider PNG b64 back to the original size (off-loop).""" + if (_model_w, _model_h) == _orig_size: + return _raw + _rs = time.perf_counter() + try: + return await asyncio.to_thread( + _restore_local_b64, _raw, _orig_size, (_model_w, _model_h) + ) + finally: + _restore_ms[0] += (time.perf_counter() - _rs) * 1000.0 + logger.debug( + "inpaint_proxy self-hosted /images/edits -> model=%r " + "size=%r steps=%r endpoint=%s", + data.get("model"), data.get("size"), + data.get("steps"), base, + ) + _prep_ms = (time.perf_counter() - _t0) * 1000.0 + + def _finish(_payload: Dict[str, Any]) -> Dict[str, Any]: + # model_ms is the provider round-trip (its own queue + + # diffusion time); prep_ms is our decode/resize; restore_ms + # is resizing the result back. total_ms is the whole op. + logger.info( + "inpaint-proxy timing prep_ms=%.1f model_ms=%.1f restore_ms=%.1f " + "total_ms=%.1f size=%s steps=%s model=%s request_id=%s", + _prep_ms, _provider_ms[0], _restore_ms[0], + (time.perf_counter() - _t0) * 1000.0, + data.get("size"), data.get("steps"), data.get("model"), + progress_request_id or "-", + ) + return _payload + + _t_post = time.perf_counter() r = await client.post(_join_checked_gallery_endpoint(base, "/images/edits"), data=data, files=files) + _provider_ms = [(time.perf_counter() - _t_post) * 1000.0] if r.status_code == 200: result = r.json() + _used_model = result.get("model") if isinstance(result, dict) else None + _used_model = _used_model or data.get("model") or "" if isinstance(result, dict) and result.get("data"): item = result["data"][0] if item.get("b64_json"): - return {"image": item["b64_json"]} + return _finish({"image": await _restore_b64(item["b64_json"]), "model": _used_model}) if item.get("url"): raw_b64 = await _fetch_result_image_b64(item["url"]) if raw_b64: - return {"image": raw_b64} + return _finish({"image": await _restore_b64(raw_b64), "model": _used_model}) if isinstance(result, dict) and result.get("image"): - return {"image": result["image"]} + return _finish({"image": await _restore_b64(result["image"]), "model": _used_model}) raise HTTPException(502, "Image edit endpoint returned no image") if r.status_code not in (404, 405): logger.warning("inpaint_proxy self-hosted edits: status %s", r.status_code) @@ -1528,16 +2055,170 @@ def setup_gallery_routes() -> APIRouter: r = await client.post(_join_checked_gallery_endpoint(base, "/images/inpaint"), json=body) if r.status_code != 200: logger.error("inpaint_proxy diffusion: status %s", r.status_code) - raise HTTPException(r.status_code, "Inpaint request failed") + detail = "Inpaint request failed" + try: + j = r.json() + if isinstance(j, dict): + detail = j.get("detail") or j.get("error") or detail + except Exception: + pass + raise HTTPException(r.status_code, f"{detail} (provider HTTP {r.status_code})") return r.json() except httpx.TimeoutException: raise HTTPException(504, "Inpaint request timed out (240s)") + except httpx.TransportError as exc: + logger.warning("inpaint_proxy: image endpoint unreachable: %s", exc) + raise HTTPException(502, f"Image server unreachable: {type(exc).__name__}") except HTTPException: raise except Exception: logger.exception("inpaint_proxy: request failed") raise HTTPException(502, "Inpaint request failed") + async def _resolve_and_decode(body: Dict[str, Any], user): + """Shared prep for the seneca single-image ops: validate endpoint + decode. + + Returns (base, api_key, image_bytes, chosen_model). The base64 decode + runs off the event loop (large images). + """ + requested_base = (body.pop("_endpoint", "") or "").rstrip("/") + if requested_base and "://" in requested_base: + from src.url_safety import check_outbound_url + ok, reason = check_outbound_url( + requested_base, + block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise HTTPException(400, f"Rejected endpoint URL: {reason}") + chosen_model = (body.pop("_model", "") or "").strip() + db = SessionLocal() + try: + base, api_key = _resolve_image_op_endpoint(db, user, requested_base, chosen_model) + finally: + db.close() + image_bytes = await asyncio.to_thread(_b64_bytes, body.get("image")) + return base, api_key, image_bytes, chosen_model + + @router.post("/api/image/replace-background") + async def replace_background(request: Request): + """Proxy seneca's subject-mask background replacement (~/v1/images/replace-background).""" + user = require_privilege(request, "can_generate_images") + body = await request.json() + base, api_key, image_bytes, chosen_model = await _resolve_and_decode(body, user) + fields: Dict[str, str] = { + "prompt": str(body.get("prompt") or ""), + "steps": str(body.get("steps") or 4), + } + if body.get("request_id"): + fields["request_id"] = str(body["request_id"]).strip() + if body.get("harmonise") is not None: + fields["harmonise"] = "true" if body.get("harmonise") else "false" + if body.get("shadow") is not None: + fields["shadow"] = "true" if body.get("shadow") else "false" + if chosen_model: + fields["model"] = chosen_model + return await _forward_seneca_image_op( + base, api_key, "/images/replace-background", image_bytes, fields, + label="replace-background", + ) + + @router.post("/api/image/outpaint") + async def outpaint(request: Request): + """Proxy seneca's canvas outpaint/extend (~/v1/images/outpaint, alias /extend).""" + user = require_privilege(request, "can_generate_images") + body = await request.json() + base, api_key, image_bytes, chosen_model = await _resolve_and_decode(body, user) + fields: Dict[str, str] = { + "prompt": str(body.get("prompt") or ""), + "steps": str(body.get("steps") or 4), + } + for key in ("ratio", "size", "context"): + if body.get(key) is not None and str(body.get(key)) != "": + fields[key] = str(body[key]) + if body.get("request_id"): + fields["request_id"] = str(body["request_id"]).strip() + if chosen_model: + fields["model"] = chosen_model + return await _forward_seneca_image_op( + base, api_key, "/images/outpaint", image_bytes, fields, + label="outpaint", timeout=_outpaint_timeout(), + ) + + @router.post("/api/image/extend") + async def outpaint_extend(request: Request): + """Alias of /api/image/outpaint (the provider exposes /outpaint as /extend too).""" + return await outpaint(request) + + @router.get("/api/image/edit-progress") + async def edit_progress(request: Request): + """Poll a running image edit's provider-side step progress. + + Contract v1 (image-op progress): the provider exposes + ``GET /v1/images/progress/{request_id}`` -> ``{stage, step, steps, + eta_s, progress}``. This proxy resolves the configured image endpoint + (owner-scoped) and returns that JSON, or ``{"available": false}`` when + the provider has no progress route yet, so the client can keep an + indeterminate bar without breaking. + """ + import httpx + user = require_privilege(request, "can_generate_images") + request_id = (request.query_params.get("request_id") or "").strip() + if not request_id: + raise HTTPException(400, "Missing request_id") + # request_id is spliced into the outbound URL, so it must be a safe token + # (no path separators/host chars). The base itself is a DB-validated row. + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,80}", request_id): + raise HTTPException(400, "Invalid request_id") + requested_base = (request.query_params.get("base") or "").strip().rstrip("/") + endpoint_id = (request.query_params.get("endpoint_id") or "").strip() + db = SessionLocal() + try: + ep = None + if endpoint_id: + ep = _visible_image_endpoint_for_base(db, endpoint_id, user) + if ep is None: + return {"available": False, "reason": "endpoint not found or disabled"} + elif requested_base: + ep = _visible_image_endpoint_for_base(db, requested_base, user) + if ep is None: + return {"available": False, "reason": "endpoint not found or disabled"} + else: + ep = _first_visible_image_endpoint(db, user) + if ep is None: + return {"available": False, "reason": "no image endpoint configured"} + base = ep.base_url.rstrip("/") + finally: + db.close() + + from src.url_safety import check_outbound_url + block_private = os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true" + ok, reason = check_outbound_url(base, block_private=block_private) + if not ok: + # CodeQL #1203: Never expose internal addresses, resolver errors or raw + # exception details to clients or logs. Use a stable diagnostic message + # without interpolating untrusted exception content or destination strings. + logger.warning("edit-progress endpoint rejected: outbound URL policy check failed") + return {"available": False, "reason": "endpoint URL rejected"} + + if not base.endswith("/v1"): + base += "/v1" + # Dynamic path segment (validated token above), so not routed through + # _join_checked_gallery_endpoint's exact-match whitelist. + url = f"{base}/images/progress/{request_id}" + try: + async with httpx.AsyncClient(timeout=10, follow_redirects=False) as client: + r = await client.get(url) + if r.status_code == 200: + data = r.json() + if isinstance(data, dict): + data.setdefault("available", True) + return data + return {"available": True, "progress": data} + return {"available": False, "status": r.status_code} + except Exception as exc: + logger.debug("edit-progress poll failed: %s", exc) + return {"available": False, "reason": "provider unreachable"} + # ---- POST /api/image/harmonize — proper img2img call ---- # Earlier version routed through inpaint with a full-white mask, but # most backends interpret "100% mask coverage" as "regenerate from @@ -1563,7 +2244,7 @@ def setup_gallery_routes() -> APIRouter: # below, so validate it first (mirrors routes/embedding_routes.py). # Local-first means loopback/LAN is allowed by default; the cloud # metadata range and non-HTTP(S) schemes are always rejected. - if requested_base: + if requested_base and "://" in requested_base: from src.url_safety import check_outbound_url ok, reason = check_outbound_url( requested_base, @@ -1590,7 +2271,7 @@ def setup_gallery_routes() -> APIRouter: # so the outbound URL never depends directly on request-body input. db = SessionLocal() try: - ep = _visible_image_endpoint_for_base(db, requested_base, user) + ep = _visible_image_endpoint_for_base(db, requested_base, user, model) if not ep: raise HTTPException(403, "Choose a registered image endpoint") base = ep.base_url.rstrip("/") @@ -1689,10 +2370,10 @@ def setup_gallery_routes() -> APIRouter: headers["Authorization"] = f"Bearer {api_key}" last_err = None - # Cold-start SDXL inpaint can take 60-90s on first request (loading - # weights to GPU). 240s gives headroom for both that and a full - # 1024×1024 inference pass on slower setups. - async with httpx.AsyncClient(timeout=240) as client: + # Cold-start image servers can take 90-110s for a full pass (and + # 60-90s more on first request while weights load to GPU). 300s gives + # headroom for both on slower setups. + async with httpx.AsyncClient(timeout=300) as client: for path, kind, payload in candidates: _effective_base = base_root if path.startswith("/sdapi") else base target = _join_checked_gallery_endpoint(_effective_base, path) @@ -1755,7 +2436,7 @@ def setup_gallery_routes() -> APIRouter: from PIL import Image, ImageFilter import base64, io - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) img = Image.open(io.BytesIO(img_bytes)).convert("RGB") # Unsharp mask: radius=2, percent=amount*200, threshold=3 @@ -1776,6 +2457,9 @@ def setup_gallery_routes() -> APIRouter: image_b64 = body.get("image") if not image_b64: raise HTTPException(400, "No image provided") + _proxied = await _route_heavy_image_op("/api/image/denoise", body, label="AI denoise") + if _proxied is not None: + return _proxied try: strength = float(body.get("strength", 0.5)) except Exception: @@ -1788,7 +2472,7 @@ def setup_gallery_routes() -> APIRouter: except ImportError: raise HTTPException(500, "Server missing a required dependency") # Decode source image (RGB; Real-ESRGAN doesn't preserve alpha). - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) src = Image.open(io.BytesIO(img_bytes)).convert("RGB") try: patch_realesrgan_torchvision_compat() @@ -1827,6 +2511,9 @@ def setup_gallery_routes() -> APIRouter: image_b64 = body.get("image") if not image_b64: raise HTTPException(400, "No image provided") + _proxied = await _route_heavy_image_op("/api/image/upscale-local", body, label="AI upscale") + if _proxied is not None: + return _proxied try: scale = int(body.get("scale", 2)) except Exception: @@ -1838,7 +2525,7 @@ def setup_gallery_routes() -> APIRouter: import numpy as np except ImportError: raise HTTPException(500, "Server missing a required dependency") - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) src = Image.open(io.BytesIO(img_bytes)).convert("RGB") try: patch_realesrgan_torchvision_compat() @@ -1865,6 +2552,74 @@ def setup_gallery_routes() -> APIRouter: logger.warning("AI upscale failed", exc_info=True) return {"error": "AI upscale failed"} + # ---- POST /api/image/generate ---- + @router.post("/api/image/generate") + async def generate_image(request: Request): + """Text-to-image generation for the editor's Quick Edit 'Image' mode. + + Generates a brand-new image from a prompt using the admin-configured + image model, returned as base64 so the client can drop it in as a layer. + """ + import json as _json + user = require_privilege(request, "can_generate_images") + try: + body = await request.json() + except Exception: + body = {} + prompt = str((body or {}).get("prompt") or "").strip() + if not prompt: + raise HTTPException(400, "A prompt is required") + try: + width = int((body or {}).get("width") or 0) + height = int((body or {}).get("height") or 0) + except (TypeError, ValueError): + width = height = 0 + if height > width: + size = "1024x1536" + elif width > height: + size = "1536x1024" + else: + size = "1024x1024" + try: + from src.ai_interaction import do_generate_image + except Exception: + raise HTTPException(502, "Image generation is unavailable") + # The editor's model picker may choose a specific (cloud) image model; + # forward it so settings defaults only apply when none is selected. + model_choice = str((body or {}).get("model") or "").strip() + payload = {"prompt": prompt, "size": size} + if model_choice: + payload["model"] = model_choice + content = _json.dumps(payload) + result = await do_generate_image(content, owner=user, incognito=True) + if not isinstance(result, dict) or result.get("error"): + msg = (result.get("error") if isinstance(result, dict) else None) or "Image generation failed" + low = str(msg).lower() + # Plain-language status codes the editor maps to fixed messages: + # 503 -> generation is switched off on the server + # 502 -> the image server could not be reached (or failed) + if "disabled" in low or "switched off" in low or "not enabled" in low: + raise HTTPException(503, "Image generation is switched off on the server at the moment.") + if "timed out" in low or "timeout" in low: + raise HTTPException(504, f"Image server timed out: {msg}") + if "connection" in low or "connect" in low or "unreachable" in low: + raise HTTPException(502, f"Image server unreachable: {msg}") + # A configured image model that cannot be resolved means its endpoint + # did not answer, i.e. the image server is down, not a bad prompt. + if "no endpoint found with image model" in low: + raise HTTPException(502, f"Image server unreachable: {msg}") + raise HTTPException(502, msg) + image_url = result.get("image_url") or "" + b64 = image_url.split(",", 1)[1] if image_url.startswith("data:") else image_url + if not b64: + raise HTTPException(502, "Image generation returned no image") + return { + "image": b64, + "prompt": prompt, + "model": result.get("image_model"), + "size": result.get("image_size") or size, + } + # ---- POST /api/image/remove-bg ---- @router.post("/api/image/mask") async def smart_mask(request: Request): @@ -1876,6 +2631,9 @@ def setup_gallery_routes() -> APIRouter: """ require_privilege(request, "can_generate_images") body = await request.json() + _proxied = await _remote_smart_mask(body) + if _proxied is not None: + return _proxied image = _b64_to_pil_image(body.get("image") or "", mode="RGB") points = body.get("points") or [] box = body.get("box") @@ -1889,6 +2647,7 @@ def setup_gallery_routes() -> APIRouter: if not points and not box: raise HTTPException(400, "Provide at least one point, box, or object text") + _image_op_log_rss("mask", "start") backend = _load_sam_backend() torch = backend["torch"] processor = backend["processor"] @@ -1985,6 +2744,8 @@ def setup_gallery_routes() -> APIRouter: } if grounded: result["grounding"] = grounded + _image_op_log_rss("mask", "end") + _unload_image_models() return result except HTTPException: raise @@ -2010,11 +2771,14 @@ def setup_gallery_routes() -> APIRouter: body = await request.json() image_b64 = body.get("image") hint_b64 = body.get("hint_mask") + _proxied = await _remote_remove_background(body) + if _proxied is not None: + return _proxied from PIL import Image import base64, io - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) img = Image.open(io.BytesIO(img_bytes)).convert("RGBA") W, H = img.size @@ -2022,7 +2786,7 @@ def setup_gallery_routes() -> APIRouter: bbox = None if hint_b64: try: - hint_bytes = base64.b64decode(hint_b64) + hint_bytes = _b64_bytes(hint_b64, label="hint_mask") hint = Image.open(io.BytesIO(hint_bytes)).convert("L") # Resize the hint to match if dimensions disagree if hint.size != img.size: @@ -2046,6 +2810,7 @@ def setup_gallery_routes() -> APIRouter: else: crop = img + _image_op_log_rss("remove-bg", "start") try: from rembg import remove from starlette.concurrency import run_in_threadpool @@ -2061,6 +2826,9 @@ def setup_gallery_routes() -> APIRouter: cut = tmp except Exception: return {"error": "No background removal model available. Install rembg: pip install rembg"} + finally: + _image_op_log_rss("remove-bg", "end") + _unload_image_models() # Compose the cropped result back into a full-size transparent canvas. if bbox: @@ -2100,11 +2868,13 @@ def setup_gallery_routes() -> APIRouter: from PIL import Image, ImageFilter, ImageEnhance import numpy as np - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) img = Image.open(io.BytesIO(img_bytes)).convert("RGB") # Try GFPGAN first (AI face restoration) try: + if not _local_image_models_enabled(): + raise ImportError("local image models disabled") from gfpgan import GFPGANer import cv2 diff --git a/routes/mcp/mcp_routes.py b/routes/mcp/mcp_routes.py index 5d14ba8c3..5a8ae9c92 100644 --- a/routes/mcp/mcp_routes.py +++ b/routes/mcp/mcp_routes.py @@ -5,6 +5,7 @@ import os import uuid import urllib.parse import html +import secrets from pathlib import Path from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import RedirectResponse, HTMLResponse @@ -12,7 +13,7 @@ import logging import httpx from core.database import McpServer, SessionLocal -from core.middleware import require_admin +from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN, require_admin from src.constants import DATA_DIR, MCP_OAUTH_DIR from src.mcp_manager import McpManager @@ -424,6 +425,11 @@ def setup_mcp_routes(mcp_manager: McpManager): Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]} """ require_admin(request) + internal_token = request.headers.get(INTERNAL_TOOL_HEADER) + if internal_token and secrets.compare_digest(internal_token, INTERNAL_TOOL_TOKEN): + # Same boundary as POST /api/tools: an agent turn must not be able + # to re-enable MCP tools an admin switched off. + raise HTTPException(403, "Change MCP tool policy in Settings > Integrations, not from chat") db = SessionLocal() try: srv = db.query(McpServer).filter(McpServer.id == server_id).first() diff --git a/routes/model_routes.py b/routes/model_routes.py index 6aec3b9d9..4535bbafe 100644 --- a/routes/model_routes.py +++ b/routes/model_routes.py @@ -7,6 +7,7 @@ import json import hashlib import ipaddress import socket +import secrets import time as _time import logging import threading @@ -473,7 +474,7 @@ def _truthy(value: str | None) -> bool: _ENDPOINT_KINDS = {"auto", "local", "api", "proxy"} _REFRESH_MODES = {"auto", "manual", "disabled"} -_MODEL_TOOL_MODES = {"none", "compact", "full"} +_MODEL_TOOL_MODES = {"none", "compact", "full", "spec"} _MODEL_TOOL_MODE_ALIASES = { "regular": "full", "odysseus_compact": "compact", @@ -553,10 +554,10 @@ def _endpoint_refresh_timeout(ep: Any, category: str) -> float: val = 0 if val > 0: return float(max(1, min(60, val))) - # llama.cpp and other local OpenAI-compatible servers can block briefly - # while warming/loading. A 2s local timeout makes working endpoints flicker - # offline before /v1/models is ready. - return 10.0 if category == "local" else 2.0 + # Keep the background refresh short so a single dead endpoint cannot stall + # the model picker. A timed-out refresh never clears a non-empty model + # cache, so a briefly-warming local server is simply retried next round. + return 2.0 def _manual_refresh_timeout(ep: Any, category: str, requested: Any = None) -> float: @@ -1511,6 +1512,9 @@ def setup_model_routes(model_discovery): _refresh_inflight = {"v": False, "done": None} # coarse single-flight guard _REFRESH_FAILURE_BASE = 300.0 _REFRESH_FAILURE_MAX = 3600.0 + # Even a user-triggered refresh must not re-probe an endpoint that just + # failed/timed out on every click; keep a short cooldown. + _REFRESH_FAILURE_MIN = 60.0 def _refresh_key(base: str, api_key: Optional[str], provider_auth_id: Optional[str] = None) -> str: # Session-backed endpoints carry no static key; include their auth id @@ -1558,14 +1562,20 @@ def setup_model_routes(model_discovery): if mode in ("manual", "disabled") and not force: return False, info fails = int(state.get("fail_count") or 0) - if fails and not force: + if fails: last_failure = float(state.get("last_failure") or 0.0) empty_local = ( not cached and category == "local" and str(getattr(ep, "id", "") or "").startswith("local-") ) - if now - last_failure < _failure_delay(fails, empty_local=empty_local): + delay = _failure_delay(fails, empty_local=empty_local) + if force: + # A user-triggered refresh gets a short cooldown instead of the + # long background backoff, but still skips a just-failed + # endpoint so repeated clicks cannot stall the picker. + delay = min(delay, _REFRESH_FAILURE_MIN) + if now - last_failure < delay: return False, info if cached and not force: interval = _endpoint_refresh_interval(ep, category) @@ -3082,6 +3092,12 @@ def setup_model_routes(model_discovery): def update_tools(body: ToolsUpdate, request: Request): """Update which tools are disabled.""" require_admin(request) + from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN + internal_token = request.headers.get(INTERNAL_TOOL_HEADER) + if internal_token and secrets.compare_digest(internal_token, INTERNAL_TOOL_TOKEN): + # Owner impersonation can carry an admin username, so check the + # authenticated transport rather than request.state.current_user. + raise HTTPException(403, "Change tool policy in Settings > Agent Tools, not from chat") settings = _load_settings() settings["disabled_tools"] = body.disabled _save_settings(settings) @@ -3089,4 +3105,10 @@ def setup_model_routes(model_discovery): router._should_refresh_endpoint = _should_refresh_endpoint router._search_endpoint_catalog = search_endpoint_catalog + # Exposed for tests: the in-memory backoff state, its key function, and the + # background refresh driver (probes endpoints in parallel with the short + # per-endpoint timeout). + router._refresh_state = _refresh_state + router._refresh_key = _refresh_key + router._refresh_caches_bg = _refresh_caches_bg return router diff --git a/routes/research/research_routes.py b/routes/research/research_routes.py index 98897a875..89f426715 100644 --- a/routes/research/research_routes.py +++ b/routes/research/research_routes.py @@ -17,6 +17,7 @@ from src.endpoint_resolver import resolve_endpoint from src.auth_helpers import _auth_disabled, get_current_user from src.owner_identity import REQUEST_SENTINEL_OWNERS from src.constants import DEEP_RESEARCH_DIR +from src import shutdown as _shutdown _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$") @@ -285,11 +286,17 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: async def research_status(session_id: str, request: Request): user = _require_user(request) _validate_session_id(session_id) + # No in-flight or persisted research for this session is a normal state + # (most sessions have none), not an error. Return HTTP 200 with an + # explicit empty status so loading a session does not log a spurious + # 404; every client treats a non-"running" status as idle anyway. + # Research owned by ANOTHER user answers identically, so the response + # does not reveal whether that research exists. if not _owns_in_memory(session_id, user): - raise HTTPException(404, "No research found for this session") + return {"status": "none"} status = research_handler.get_status(session_id) if status is None: - raise HTTPException(404, "No research found for this session") + return {"status": "none"} try: source_state = research_handler.get_source_state(session_id) if isinstance(source_state, str) and source_state: @@ -657,7 +664,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: raise HTTPException(404, "No research found for this session") async def _generate(): last_payload = None - while True: + while not _shutdown.is_shutting_down(): status = research_handler.get_status(session_id) if status is None: yield f"data: {json.dumps({'status': 'not_found'})}\n\n" diff --git a/routes/session_routes.py b/routes/session_routes.py index 9f6cc3659..1c903f194 100644 --- a/routes/session_routes.py +++ b/routes/session_routes.py @@ -1320,7 +1320,7 @@ def setup_session_routes( after Phase 1 — used by the "Tidy (no AI)" UI affordance so users can clean junk without spending tokens. """ - from src.llm_core import llm_call + from src.llm_core import llm_call, llm_call_with_fallback from src.utility_effort import effort_for_call user = effective_user(request) single_user_mode = not user and _auth_disabled() @@ -1467,7 +1467,7 @@ def setup_session_routes( return {"status": "skipped", "reason": "No unfiled sessions to sort"} # Pick an endpoint — prefer admin-configured task endpoint - from src.task_endpoint import resolve_task_endpoint + from src.task_endpoint import resolve_task_endpoint, resolve_task_candidates url, model, headers = resolve_task_endpoint(owner=user) if not url: url, model, headers = _pick_endpoint_for_sort(owner=user) @@ -1493,9 +1493,12 @@ def setup_session_routes( # 16384 (was 4096): with many chats the folder JSON is large, and a # reasoning model spends tokens thinking first — 4096 truncated the # JSON mid-output, so it never parsed ("invalid JSON for auto-sort"). - raw = llm_call(url, model, [{"role": "user", "content": prompt}], - temperature=0.3, max_tokens=16384, headers=headers, timeout=120, - reasoning_effort=effort_for_call(url, model, user)) + raw = llm_call_with_fallback( + resolve_task_candidates(fallback_url=url, fallback_model=model, + fallback_headers=headers, owner=user), + [{"role": "user", "content": prompt}], + temperature=0.3, max_tokens=16384, timeout=120, + reasoning_effort=effort_for_call(url, model, user)) logger.info(f"Auto-sort raw response ({len(raw)} chars): {raw[:300]}") # Extract JSON from response — handle markdown fences, leading text, # reasoning-model blocks, and trailing commas. diff --git a/routes/shell_routes.py b/routes/shell_routes.py index 8e2e969e0..b4684f779 100644 --- a/routes/shell_routes.py +++ b/routes/shell_routes.py @@ -27,6 +27,7 @@ from src.host_docker_access import ( from src.optional_deps import prepare_optional_dependency_import from src.auth_helpers import _auth_disabled from src import process_lifecycle +from src import shutdown as _shutdown # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist # on Windows, so importing them unconditionally crashed app startup there @@ -1025,7 +1026,7 @@ async def _generate_tmux(cmd: str, request: Request): lines_sent = 0 exit_code = None - while True: + while not _shutdown.is_shutting_down(): # Check client disconnect if await request.is_disconnected(): # tmux keeps running — that's the whole point @@ -1142,7 +1143,7 @@ async def _generate_win_detached(cmd: str, request: Request): lines_sent = 0 exit_code = None - while True: + while not _shutdown.is_shutting_down(): if await request.is_disconnected(): yield f"data: {json.dumps({'stream': 'stdout', 'data': f'Disconnected. Background job {session_id} continues running.'})}\n\n" return @@ -1267,7 +1268,7 @@ def setup_shell_routes() -> APIRouter: """Read chunks, split on \\n or \\r for progress bar support.""" try: buf = b"" - while True: + while not _shutdown.is_shutting_down(): chunk = await stream.read(4096) if not chunk: if buf: diff --git a/routes/upload_routes.py b/routes/upload_routes.py index 347da487e..081c01d5c 100644 --- a/routes/upload_routes.py +++ b/routes/upload_routes.py @@ -435,11 +435,11 @@ def setup_upload_routes(upload_handler): logger.error(f"Failed to get upload stats: {e}") raise HTTPException(500, "Failed to get upload statistics") - @router.get("/{file_id}") + @router.api_route("/{file_id}", methods=["GET", "HEAD"]) async def download_file(request: Request, file_id: str, thumb: int = 0): - """Serve an uploaded file by its ID. `?thumb=1` returns a small cached - JPEG thumbnail for images (used by chat attachment previews) so the - client isn't downloading the full-resolution photo just to show it tiny.""" + """Serve an uploaded file by its ID or probe via HEAD. `?thumb=1` returns + a small cached JPEG thumbnail for images (used by chat attachment previews) + so the client isn't downloading the full-resolution photo just to show it tiny.""" if not upload_handler.validate_upload_id(file_id): raise HTTPException(400, "Invalid file ID") import mimetypes as _mt @@ -458,29 +458,42 @@ def setup_upload_routes(upload_handler): mime = (info or {}).get("mime") or _mt.guess_type(path)[0] or "application/octet-stream" from fastapi.responses import FileResponse # Downscaled thumbnail for image previews — generated once and cached. + # Skip generation during HEAD probe to avoid loading/processing whole image. if thumb and mime.startswith("image/"): - try: - from PIL import Image, ImageOps - thumb_dir = os.path.join(_upload_root(), ".thumbs") - os.makedirs(thumb_dir, exist_ok=True) - thumb_path = os.path.join(thumb_dir, file_id + ".jpg") - if (not os.path.exists(thumb_path) - or os.path.getmtime(thumb_path) < os.path.getmtime(path)): - im = Image.open(path) - # iPhone / camera JPEGs encode rotation in EXIF rather than - # the pixel data. Browsers honour that on the original via - # image-orientation:from-image, but PIL strips EXIF when it - # saves the JPEG thumb, leaving the pixels sideways. Bake - # the rotation into the pixels before thumbnailing. - im = ImageOps.exif_transpose(im) - im.thumbnail((320, 320)) - if im.mode not in ("RGB", "L"): - im = im.convert("RGB") - im.save(thumb_path, "JPEG", quality=80) - return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS) - except Exception as e: - logger.warning(f"Thumbnail generation failed for {file_id}: {e}") - # Fall through to the full image. + thumb_dir = os.path.join(_upload_root(), ".thumbs") + thumb_path = os.path.join(thumb_dir, file_id + ".jpg") + if request.method == "HEAD": + if (os.path.isfile(thumb_path) + and os.path.getmtime(thumb_path) >= os.path.getmtime(path)): + return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS) + # GET may generate a JPEG or fall back to the original on failure. + # Neither representation is known without decoding the image. HEAD + # must not advertise the original/stale file's headers as a thumb. + from fastapi.responses import Response + response = Response(status_code=200, headers=UPLOAD_RESPONSE_HEADERS) + del response.headers["content-length"] + return response + else: + try: + from PIL import Image, ImageOps + os.makedirs(thumb_dir, exist_ok=True) + if (not os.path.exists(thumb_path) + or os.path.getmtime(thumb_path) < os.path.getmtime(path)): + im = Image.open(path) + # iPhone / camera JPEGs encode rotation in EXIF rather than + # the pixel data. Browsers honour that on the original via + # image-orientation:from-image, but PIL strips EXIF when it + # saves the JPEG thumb, leaving the pixels sideways. Bake + # the rotation into the pixels before thumbnailing. + im = ImageOps.exif_transpose(im) + im.thumbnail((320, 320)) + if im.mode not in ("RGB", "L"): + im = im.convert("RGB") + im.save(thumb_path, "JPEG", quality=80) + return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS) + except Exception as e: + logger.warning(f"Thumbnail generation failed for {file_id}: {e}") + # Fall through to the full image. return FileResponse( path, media_type=mime, diff --git a/scripts/generate_env_reference.py b/scripts/generate_env_reference.py index 06a8a4c64..c83c992f4 100644 --- a/scripts/generate_env_reference.py +++ b/scripts/generate_env_reference.py @@ -914,6 +914,78 @@ VARIABLE_NOTES: dict[str, tuple[str, str, str]] = { "Overrides the source commit reported for runtime provenance, for builds " "that ship without a git directory.", ), + + # -- Photo editor and image operations ----------------------------------- + "ODYSSEUS_EDITOR_AGENT_BASES": ( + "Model routing and providers", USER, + "Comma-separated base URLs for the image-editor agent backends; a single " + "base URL in the model endpoint config wins over this list.", + ), + "ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS": ( + "Model routing and providers", USER, + "Set to 1, true, yes or on to allow local image-generation and editing " + "backends to be reachable as image providers.", + ), + "ODYSSEUS_IMAGE_OPS_BASE": ( + "Model routing and providers", USER, + "Base URL of the image-operations backend (inpaint, outpainting and style " + "transfer); a trailing slash is stripped before use.", + ), + "ODYSSEUS_IMAGE_EDIT_MAX_SIDE": ( + "Upload and media limits", USER, + "Maximum pixel side (width or height) an edited image is downscaled to " + "before it is sent to the image backend.", + ), + "ODYSSEUS_IMAGE_EDIT_MAX_STEPS": ( + "Upload and media limits", USER, + "Maximum number of iterative edit steps the image backend may perform in " + "a single request.", + ), + "ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT": ( + "Upload and media limits", USER, + "Seconds the outpainting proxy waits for the image backend before giving " + "up on a single request.", + ), + "ODYSSEUS_JEFF_": ("Agent loop and tool execution", INTERNAL, "Prefix for per-job decision adapter opt-ins; each job is disabled by default."), + "ODYSSEUS_JEFF_TIMEOUT": ("Agent loop and tool execution", INTERNAL, "Timeout in seconds for a decision adapter request."), + "ODYSSEUS_JEFF_URL": ("Agent loop and tool execution", INTERNAL, "Base URL of the optional decision adapter service."), + "ODYSSEUS_PRIVATE_BROWSER_LEGACY": ("Agent loop and tool execution", INTERNAL, "Explicit opt-in to the legacy browser compatibility transport; disabled by default."), + "ODYSSEUS_SPEC_BROWSER_FALLBACK": ("Agent loop and tool execution", INTERNAL, "Optional browser routing fallback: mcp, web, or 0 to disable."), + "ODYSSEUS_SPEC_MODE": ("Agent loop and tool execution", INTERNAL, "Spec routing mode for explicitly configured models, including strict and driver."), + "ODYSSEUS_SPEC_ROUTER_MIN_CONF": ("Agent loop and tool execution", INTERNAL, "Minimum tool-selection confidence accepted from the router."), + "ODYSSEUS_SPEC_ROUTER_MODEL": ("Agent loop and tool execution", INTERNAL, "Decision adapter model used for tool routing."), + "ODYSSEUS_SPEC_ROUTER_SECOND": ("Agent loop and tool execution", INTERNAL, "Minimum confidence for adding a secondary tool candidate."), + "ODYSSEUS_SPEC_ROUTER_TIMEOUT": ("Agent loop and tool execution", INTERNAL, "Time budget in seconds for a router decision."), + "ODYSSEUS_SPEC_ROUTER_URL": ("Agent loop and tool execution", INTERNAL, "Base URL of the optional Spec router; an empty URL disables requests."), + "ODYSSEUS_SPEC_THINK_MIN_P": ("Agent loop and tool execution", INTERNAL, "Minimum router confidence for enabling thinking."), + "ODYSSEUS_SPEC_THINK_MODEL": ("Agent loop and tool execution", INTERNAL, "Optional thinking decision adapter model, falling back to the tool router model."), + "ODYSSEUS_SPEC_THINK_QUESTION": ("Agent loop and tool execution", INTERNAL, "Thinking classifier question format: binary or kind5."), + "ODYSSEUS_SPEC_THINK_ROUTER": ("Agent loop and tool execution", INTERNAL, "Enable router thinking decisions for explicitly configured Spec models."), + "ODYSSEUS_SPEC_TOOL_FORMAT": ("Agent loop and tool execution", INTERNAL, "Use names for compact router tool signatures, otherwise include descriptions."), + "ODYSSEUS_SPEC_TRIM_PROMPT": ("Agent loop and tool execution", INTERNAL, "Enable prompt trimming for the selected Spec tool menu."), + "ODYSSEUS_VISION_MAX_SIDE": ("Upload and media limits", USER, "Maximum pixel side for image attachments sent directly to a vision model."), + "ODYSSEUS_VISION_MODELS": ("Model routing and providers", USER, "Comma-separated model-name fragments that explicitly support inline images."), + "ODYSSEUS_VISION_PROBE": ("Model routing and providers", USER, "Allow a cached vision-capability probe against local model endpoints."), + "ODYSSEUS_SPEC_DIRECT_MODE": ( + "Agent loop and tool execution", INTERNAL, + "Controls how the spec router answers high-confidence direct-answer turns; " + "either tool_choice_none (default) or empty_menu.", + ), + "ODYSSEUS_SPEC_DIRECT_NOTOOLS": ( + "Agent loop and tool execution", INTERNAL, + "Set to 1, true, yes or on to enable the spec router's direct-answer " + "no-tools path for high-confidence answer_directly decisions.", + ), + "ODYSSEUS_SPEC_DIRECT_MIN_P": ( + "Agent loop and tool execution", INTERNAL, + "Minimum confidence probability a spec-router direct-answer decision must " + "reach before the no-tools path is taken.", + ), + "ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT": ( + "Deployment and first run", INTERNAL, + "Seconds the server waits for in-flight requests and background loops to " + "finish during a graceful shutdown.", + ), } diff --git a/scripts/sanitize_trace_artifacts.py b/scripts/sanitize_trace_artifacts.py new file mode 100644 index 000000000..ea975b3b5 --- /dev/null +++ b/scripts/sanitize_trace_artifacts.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Sanitize Playwright trace archives to strip sensitive credentials and tokens. + +Structured, format-aware fail-closed security: +Replaces naive regex-based replacement with structured JSON / HAR parsing, +recursive key/value redaction, URL credential scrubbing, DOM snapshot sanitization, +safe source code masking, and strict fail-closed purge on corruption or error. +""" +from __future__ import annotations + +import json +import logging +import os +import re +import sys +import urllib.parse +import zipfile +from typing import Any, Dict, List, Set + +logger = logging.getLogger(__name__) + +BINARY_IMAGE_EXTENSIONS = ('.png', '.jpg', '.jpeg', '.webp', '.gif', '.ico') + +SENSITIVE_KEY_PATTERN = re.compile( + r'(?i)(password|passwd|secret|token|api_key|apikey|access_token|refresh_token|private_key|auth_token|client_secret|session|session_id|sid|csrf|xsrf|jwt|credential|authorization)' +) + +SENSITIVE_PARAM_NAMES = frozenset({ + "token", "session", "auth", "password", "passwd", "key", "secret", "api_key", + "apikey", "access_token", "refresh_token", "jwt", "sid", "code", "sig", + "signature", "client_secret", "credential" +}) + +SENSITIVE_HEADER_NAMES = frozenset({ + "authorization", "proxy-authorization", "cookie", "set-cookie", "x-api-key", + "api-key", "x-auth-token", "x-access-token" +}) + + +def sanitize_url(url: str) -> str: + """Sanitize query parameters and authority credentials in URLs.""" + if not isinstance(url, str) or not url.strip(): + return url + try: + parts = urllib.parse.urlsplit(url) + except Exception: + return url + + netloc = parts.netloc + if "@" in netloc: + userinfo, host = netloc.split("@", 1) + if ":" in userinfo: + user, _ = userinfo.split(":", 1) + netloc = f"{user}:[REDACTED]@{host}" + else: + netloc = f"[REDACTED]@{host}" + + query = parts.query + if query: + q_pairs = urllib.parse.parse_qsl(query, keep_blank_values=True) + sanitized_pairs = [] + for k, v in q_pairs: + k_lower = k.lower().replace("-", "_") + if k_lower in SENSITIVE_PARAM_NAMES or SENSITIVE_KEY_PATTERN.search(k_lower): + sanitized_pairs.append((k, "[REDACTED]")) + else: + sanitized_pairs.append((k, v)) + query = urllib.parse.urlencode(sanitized_pairs) + + return urllib.parse.urlunsplit((parts.scheme, netloc, parts.path, query, parts.fragment)) + + +def sanitize_body_text(text: str) -> str: + """Sanitize request bodies formatted as JSON or URL-encoded form data.""" + if not isinstance(text, str) or not text.strip(): + return text + + # 1. Try JSON + trimmed = text.strip() + if (trimmed.startswith("{") and trimmed.endswith("}")) or (trimmed.startswith("[") and trimmed.endswith("]")): + try: + parsed = json.loads(trimmed) + sanitized = sanitize_structured_obj(parsed) + return json.dumps(sanitized, ensure_ascii=False) + except Exception: + pass + + # 2. Try URL-encoded form data + if "=" in trimmed and not trimmed.startswith("<"): + try: + pairs = urllib.parse.parse_qsl(trimmed, keep_blank_values=True) + if pairs and any(k.lower() in SENSITIVE_PARAM_NAMES or SENSITIVE_KEY_PATTERN.search(k) for k, _ in pairs): + out_pairs = [] + for k, v in pairs: + k_lower = k.lower().replace("-", "_") + if k_lower in SENSITIVE_PARAM_NAMES or SENSITIVE_KEY_PATTERN.search(k_lower): + out_pairs.append((k, "[REDACTED]")) + else: + out_pairs.append((k, v)) + return urllib.parse.urlencode(out_pairs) + except Exception: + pass + + return text + + +def sanitize_structured_obj(obj: Any) -> Any: + """Recursively sanitize Python objects decoded from JSON or HAR payloads.""" + if isinstance(obj, dict): + new_dict: Dict[str, Any] = {} + + # Check for header entry with inverted or standard keys: {"name": ..., "value": ...} + name_val = obj.get("name") + val_val = obj.get("value") + if isinstance(name_val, str) and isinstance(val_val, str): + name_lower = name_val.lower().strip() + if name_lower in SENSITIVE_HEADER_NAMES or SENSITIVE_KEY_PATTERN.search(name_lower): + for k, v in obj.items(): + if k == "value": + new_dict[k] = "[REDACTED]" + else: + new_dict[k] = sanitize_structured_obj(v) + return new_dict + + # Process standard dictionary keys + for k, v in obj.items(): + k_lower = str(k).lower().replace("-", "_") + + # Action parameters in Playwright traces (e.g. page.fill, page.type) + if k == "params" and isinstance(v, dict): + sanitized_params = sanitize_structured_obj(v) + # If this action involves fill, type, or sensitive inputs, redact value/text + if "value" in sanitized_params: + sanitized_params["value"] = "[REDACTED]" + if "text" in sanitized_params: + sanitized_params["text"] = "[REDACTED]" + new_dict[k] = sanitized_params + continue + + # DOM snapshot nodes + if k == "attributes" and isinstance(v, (dict, list)): + if isinstance(v, dict): + attr_dict = dict(v) + type_val = str(attr_dict.get("type") or "").lower() + if type_val == "password" or "password" in str(attr_dict.get("name") or "").lower(): + if "value" in attr_dict: + attr_dict["value"] = "[REDACTED]" + new_dict[k] = attr_dict + continue + elif isinstance(v, list): + # List of attribute strings or pairs + new_list = [] + is_pw = any("password" in str(x).lower() for x in v) + for x in v: + if is_pw and isinstance(x, str) and not x.startswith("type="): + new_list.append("[REDACTED]") + else: + new_list.append(sanitize_structured_obj(x)) + new_dict[k] = new_list + continue + + # Request postData / text + if k in ("postData", "postDataText") and isinstance(v, str): + new_dict[k] = sanitize_body_text(v) + continue + + # URL fields + if k == "url" and isinstance(v, str): + new_dict[k] = sanitize_url(v) + continue + + # Cookie arrays + if k == "cookies" and isinstance(v, list): + sanitized_cookies = [] + for c in v: + if isinstance(c, dict): + c_copy = dict(c) + if "value" in c_copy: + c_copy["value"] = "[REDACTED]" + sanitized_cookies.append(c_copy) + else: + sanitized_cookies.append(sanitize_structured_obj(c)) + new_dict[k] = sanitized_cookies + continue + + # Direct sensitive key matches + if SENSITIVE_KEY_PATTERN.search(k_lower) and isinstance(v, str): + new_dict[k] = "[REDACTED]" + continue + + new_dict[k] = sanitize_structured_obj(v) + + return new_dict + + elif isinstance(obj, list): + return [sanitize_structured_obj(item) for item in obj] + + elif isinstance(obj, str): + # Check for inline credentials/tokens + if obj.startswith("Bearer ") or obj.startswith("bearer "): + return "Bearer [REDACTED]" + if obj.startswith("Basic ") or obj.startswith("basic "): + return "Basic [REDACTED]" + return obj + + return obj + + +def sanitize_jsonl_content(data: bytes) -> bytes: + """Sanitize newline-delimited JSON Lines (.trace, .network, .stacks).""" + lines = data.split(b"\n") + out_lines = [] + for line in lines: + if not line.strip(): + out_lines.append(line) + continue + try: + parsed = json.loads(line.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + # Not valid JSON, keep as-is + out_lines.append(line) + continue + sanitized = sanitize_structured_obj(parsed) + out_lines.append(json.dumps(sanitized, ensure_ascii=False).encode("utf-8")) + return b"\n".join(out_lines) + + +def sanitize_trace_archive(zp: str) -> bool: + """Sanitize a single zip archive in a strict fail-closed manner. + + If the archive is malformed, partial, unreadable, or encounters an exception + during processing, the archive is permanently deleted so that raw traces + cannot leak. + Returns True if successfully sanitized, False if deleted due to error. + """ + if not os.path.exists(zp): + return False + + temp_zp = zp + ".tmp" + try: + if os.path.getsize(zp) == 0: + raise zipfile.BadZipFile("Empty archive") + + with zipfile.ZipFile(zp, "r") as zin: + infolist = zin.infolist() + if not infolist: + raise zipfile.BadZipFile("Archive has no entries") + + corrupt_file = zin.testzip() + if corrupt_file is not None: + raise zipfile.BadZipFile(f"Corrupt entry detected: {corrupt_file}") + + with zipfile.ZipFile(temp_zp, "w", compression=zipfile.ZIP_DEFLATED) as zout: + for item in infolist: + filename_lower = item.filename.lower() + + # 1. Binary image screenshots/diagnostics: preserve as-is + if filename_lower.endswith(BINARY_IMAGE_EXTENSIONS): + content = zin.read(item.filename) + zout.writestr(item, content) + continue + + # 2. Embedded test source code (e.g. resources/src@... or .js/.ts): + # Mask safely without syntax corruption! + if "src@" in filename_lower or filename_lower.endswith((".js", ".ts", ".mjs", ".py")): + safe_placeholder = b"// [SOURCE CODE REDACTED FOR TRACE CONFIDENTIALITY]\n" + zout.writestr(item, safe_placeholder) + continue + + # 3. JSON Lines and structured JSON/HAR traces + content = zin.read(item.filename) + if filename_lower.endswith((".trace", ".network", ".stacks")): + sanitized_content = sanitize_jsonl_content(content) + zout.writestr(item, sanitized_content) + elif filename_lower.endswith((".json", ".har")): + try: + parsed = json.loads(content.decode("utf-8")) + sanitized = sanitize_structured_obj(parsed) + zout.writestr(item, json.dumps(sanitized, ensure_ascii=False).encode("utf-8")) + except Exception: + # If invalid JSON, treat as text line-by-line + zout.writestr(item, sanitize_jsonl_content(content)) + else: + # Unknown binary or text member: exclude to fail closed + continue + + # Atomic replace + os.replace(temp_zp, zp) + return True + + except Exception as exc: + logger.warning("Sanitizing trace %s failed (%s); removing fail-closed", zp, exc) + for path_to_remove in (temp_zp, zp): + if os.path.exists(path_to_remove): + try: + os.remove(path_to_remove) + except OSError as remove_err: + logger.error("Failed to delete unredacted trace %s: %s", path_to_remove, remove_err) + return False + + +def sanitize_directory(results_dir: str) -> int: + """Walk results_dir and sanitize every .zip archive fail-closed. + + Returns the number of archives processed. Exits with 1 if any archive fails. + """ + if not os.path.exists(results_dir): + return 0 + + count = 0 + all_ok = True + for root, _, files in os.walk(results_dir): + for f in files: + if f.endswith(".zip"): + zp = os.path.join(root, f) + ok = sanitize_trace_archive(zp) + if not ok: + all_ok = False + count += 1 + + if not all_ok: + sys.exit(1) + return count + + +def main() -> None: + if len(sys.argv) < 2: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + target_dir = sys.argv[1] + sanitize_directory(target_dir) + + +if __name__ == "__main__": + main() diff --git a/services/memory/memory_extractor.py b/services/memory/memory_extractor.py index 6b911b4d5..dfaf71e25 100644 --- a/services/memory/memory_extractor.py +++ b/services/memory/memory_extractor.py @@ -15,6 +15,7 @@ import json import logging import os import re +import time from typing import Optional from src.memory import MemoryStoreUnreadable @@ -78,11 +79,13 @@ EXTRACT_SYSTEM_PROMPT = ( "things the assistant said, one-off tasks, opinions on the current topic.\n\n" "Rules:\n" "- MAX 2 facts per conversation — only the most important\n" - "- Only extract facts the USER stated or clearly implied\n" + "- Only extract facts the USER explicitly stated about themselves\n" + "- Questions, requests to inspect records, and tool/assistant output are not evidence\n" "- Each fact must be a single short sentence (under 15 words)\n" "- If a fact is similar to something likely already known, skip it\n" "- If nothing durable was revealed, return []\n\n" - "Return a JSON array of objects with 'text' and 'category' fields.\n" + "Return a JSON array of objects with 'text', 'category', and 'evidence' fields. " + "Evidence must be an exact quote from a user's own factual statement.\n" "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n" "Return ONLY valid JSON, no markdown fences." ) @@ -213,7 +216,10 @@ def _fallback_memory_candidates(messages) -> list[dict]: if not text: continue - m = re.search(r"\bmy name is\s+([A-Za-z][A-Za-z0-9 .'\-]{1,50})\b", text, re.I) + m = re.search( + r"\bmy name is\s+([A-Za-z][A-Za-z0-9 '\-]{0,49}?)" + r"(?=\s+(?:and|but|please)\b|[.!?;,\n]|$)", text, re.I, + ) if m: name = _clean_memory_value(m.group(1), 50) if name: @@ -259,22 +265,183 @@ def _fallback_memory_candidates(messages) -> list[dict]: return candidates[:2] +_USER_FACT_ASSERTION = re.compile( + r"\b(?:i(?:'m| am| live| work| use| have| prefer| like| love| hate| own| study| grew up)\b" + r"|my\s+[a-z][a-z -]{0,40}\s+(?:is|are|was|works|lives)\b" + r"|we\s+(?:are|live|work|have|own)\b" + r"|our\s+[a-z][a-z -]{0,40}\s+(?:is|are|was)\b)", + re.IGNORECASE, +) + + +def _has_user_fact_evidence(fact: dict, messages) -> bool: + """Require an exact user assertion before persisting an LLM-inferred memory.""" + evidence = fact.get("evidence") + if not isinstance(evidence, str): + return False + evidence = re.sub(r"\s+", " ", evidence).strip() + if len(evidence.split()) < 3 or not _USER_FACT_ASSERTION.match(evidence): + return False + for message in messages: + if _message_role(message) != "user": + continue + utterance = re.sub(r"\s+", " ", _message_text(message)).strip() + position = utterance.casefold().find(evidence.casefold()) + if position < 0: + continue + # Judge the clause containing the quote, so a later question does not + # erase an earlier factual statement in the same user turn. + clause_start = max( + utterance.rfind(mark, 0, position) for mark in (".", "?", "!", ";") + ) + 1 + clause_end = min( + [end for mark in (".", "?", "!", ";") + if (end := utterance.find(mark, position)) >= 0] or [len(utterance)] + ) + clause = utterance[clause_start:clause_end].strip() + # A question about one's records is not a statement of the answer. + if utterance[clause_end:clause_end + 1] == "?" or re.match( + r"^(?:who|what|where|when|why|how|do|does|did|can|could|would|" + r"show|list|find|search|tell|summarize|open|read)\b", + clause, re.IGNORECASE, + ): + continue + return True + return False + + +_TOKEN_RE = re.compile(r"[a-z0-9]+") + +# Function words carry no distinguishing content. Dropping them before +# comparing two facts keeps punctuation and harmless rephrasing from defeating +# the duplicate check. +_STOPWORDS = frozenset( + "a an the and or but if then of to in on at for with by from as is are was " + "were be been being do does did has have had it its this that these those " + "there here user users someone somebody they them their he she his her him i " + "we you your my our me us who whom whose what when where why how also " + "currently now still about into over under again more most very can could " + "may might must will would should".split() +) + +# Words that frame a name/username statement; excluded when reading the value +# so the actual name tokens can be compared. +_NAME_TEMPLATE_WORDS = frozenset( + "name names named called call appears appear seems seem the user users his " + "her their it is are was were be been being username usernames login " + "account handle".split() +) + +_NAME_KEYWORDS = frozenset({"name", "named", "called"}) +_USERNAME_KEYWORDS = frozenset({"username", "login"}) + + +def _content_tokens(text: str) -> set: + """Lowercased content tokens with punctuation and stopwords removed.""" + toks = _TOKEN_RE.findall((text or "").lower()) + return {t for t in toks if t not in _STOPWORDS and len(t) >= 2} + + +def _name_slot(text: str): + """Return (slot, value_tokens) when text states a name/username, else None. + + Tokens are scanned so "username" is never mistaken for "name", and the + first keyword decides the slot (a sentence about a server *named* X that + also mentions a *username* Y is a 'name' fact, not a username fact).""" + words = _TOKEN_RE.findall((text or "").lower()) + slot = None + for w in words: + if w in _NAME_KEYWORDS: + slot = "name" + break + if w in _USERNAME_KEYWORDS: + slot = "username" + break + if not slot: + return None + value = { + w for w in words + if w not in _NAME_TEMPLATE_WORDS and w not in _STOPWORDS and len(w) >= 2 + } + return (slot, value) if value else None + + +def _name_slot_tokens(text: str): + slot = _name_slot(text) + return slot[1] if slot else None + + +def _name_vals_close(a: set, b: set) -> bool: + """True when two name values state the same name: identical, or one is a + strict refinement of the other where the larger value is a clean name + (<=4 alphabetic tokens). Guards against merging a username into a fact + like 'a server named Odysseus ... with username pewds'.""" + if a == b: + return True + if not a or not b: + return False + bigger = a if len(a) >= len(b) else b + clean = len(bigger) <= 4 and all(t.isalpha() for t in bigger) + return clean and (a <= b or b <= a) + + def _is_text_duplicate(new_text: str, existing: list, threshold: float = 0.6) -> bool: - """Check if new_text is too similar to any existing memory (Jaccard similarity).""" - new_tokens = set(new_text.lower().split()) + """Check if new_text states the same fact as an existing memory. + + Three tests, cheapest first: symmetric Jaccard over content tokens, + containment (one fact is a strict refinement of the other), and a + name/username-slot match. The naive whole-string comparison this replaces + kept punctuation (so "Alex." != "alex") and missed short-vs-full names + ("Felix" vs "FELIX ARVID ULF KJELLBERG" scored 0.43). + """ + new_tokens = _content_tokens(new_text) if not new_tokens: return False + new_slot = _name_slot(new_text) for entry in _memory_dicts(existing): - old_tokens = set(entry.get("text", "").lower().split()) + old_text = entry.get("text", "") + old_tokens = _content_tokens(old_text) if not old_tokens: continue intersection = new_tokens & old_tokens + if not intersection: + continue union = new_tokens | old_tokens if len(intersection) / len(union) >= threshold: return True + # Containment: one fact is a strict refinement of the other. Guard on + # the shorter side so two-token facts ("Likes Python") are not merged + # into longer sentences that merely start the same way. + if len(intersection) >= 3 and len(intersection) / min(len(new_tokens), len(old_tokens)) >= 0.85: + return True + # Name slot: both state the same kind of name and the values match + # (short vs full name, or the same name behind different phrasing). + old_slot = _name_slot(old_text) + if new_slot and old_slot and new_slot[0] == old_slot[0]: + if _name_vals_close(new_slot[1], old_slot[1]): + return True return False +def _name_enrichment_target(new_text: str, existing: list): + """Return an existing entry whose name value is a strict subset of the + new fact's, so the fuller name replaces it in place; else None.""" + ns = _name_slot(new_text) + if not ns: + return None + slot, new_val = ns + best = None + for entry in _memory_dicts(existing): + old_slot = _name_slot(entry.get("text", "")) + if not old_slot or old_slot[0] != slot: + continue + old_val = old_slot[1] + if old_val < new_val and _name_vals_close(old_val, new_val) and len(new_val) <= 4 and all(t.isalpha() for t in new_val): + if best is None or len(entry.get("text", "")) < len(best.get("text", "")): + best = entry + return best + + def _parse_extraction_json(raw: str) -> list: """Parse the extraction LLM's reply into a list of facts, tolerating reasoning-model noise. @@ -420,8 +587,13 @@ async def extract_and_store( if not isinstance(facts, list): facts = [] + # The extraction model sees assistant/tool text and can mistake a + # failed lookup or a one-off request for a durable personal fact. + # Only its claims with a quoted user assertion may enter memory. + facts = [fact for fact in facts if isinstance(fact, dict) + and _has_user_fact_evidence(fact, stripped_recent)] if fallback_facts: - facts = list(facts) + fallback_facts + facts.extend(fallback_facts) if not facts: logger.info("Auto memory extraction ran: 0 candidates") @@ -456,6 +628,14 @@ async def extract_and_store( if not fact_text or len(fact_text) < 5: continue + # A fuller owner-specific name must not rewrite a shared legacy + # entry, but that less-specific entry must not suppress it either. + legacy_refinements = { + id(entry) for entry in existing + if _owner and entry.get("owner") is None + and _name_enrichment_target(fact_text, [entry]) is entry + } + # Dedup: check vector similarity first (fast), then exact text match. # A runtime embedding/ChromaDB failure (backend OOM, model evicted, # remote endpoint down) must not abort the whole batch — fall through @@ -477,12 +657,38 @@ async def extract_and_store( # silently dropped. Mirror the owner predicate used by the # text dedup below; cross-tenant/stale matches fall through. _match = next((e for e in existing if e.get("id") == existing_id), None) - if _match is not None and (_match.get("owner") == _owner or _match.get("owner") is None): + if (_match is not None and id(_match) not in legacy_refinements + and (_match.get("owner") == _owner or _match.get("owner") is None)): logger.debug(f"Memory dedup (vector): '{fact_text[:50]}' matches {existing_id}") continue # Text dedup fallback: exact match + fuzzy similarity - user_existing = [e for e in existing if e.get("owner") == _owner or e.get("owner") is None] if _owner else existing + user_existing = [e for e in existing + if (e.get("owner") == _owner or e.get("owner") is None) + and id(e) not in legacy_refinements] if _owner else existing + + # Name enrichment: if an existing name fact is a strict prefix of + # this more complete one, upgrade it in place instead of adding a + # near-duplicate ("name is Felix" -> "name is Felix Arvid Ulf + # Kjellberg"). The vector store upserts by id, so re-adding is safe. + # Legacy unowned entries may be visible to multiple owners; never + # rewrite one with a private fact from the current owner. + _enrich = _name_enrichment_target( + fact_text, [e for e in user_existing if e.get("owner") == _owner] + ) + if _enrich is not None: + _enrich["text"] = fact_text + _enrich["timestamp"] = int(time.time()) + if category: + _enrich["category"] = category + if memory_vector and memory_vector.healthy: + try: + memory_vector.add(_enrich["id"], fact_text) + except Exception as e: + logger.warning(f"Memory vector update failed for {_enrich['id']}: {e}") + added += 1 + continue + if memory_manager.find_duplicates(fact_text, user_existing): continue # Fuzzy text similarity check (catches rephrased duplicates when vector index is unavailable) diff --git a/src/activity.py b/src/activity.py new file mode 100644 index 000000000..92c4098e5 --- /dev/null +++ b/src/activity.py @@ -0,0 +1,61 @@ +"""In-flight activity counters (heavy image ops) for safe restarts/observability. + +A deploy restart kills any in-flight long image op (a diffusion edit is 60-120 s), +so the operator needs to know whether the process is busy before issuing +``systemctl --user restart``. These counters are process-local and cheap. +""" +from __future__ import annotations + +import threading +import time +from contextlib import contextmanager +from typing import Dict + +_lock = threading.Lock() +_image_ops = 0 +_image_ops_by_name: Dict[str, int] = {} +_started_at = time.time() + + +def begin_image_op(name: str = "image") -> None: + global _image_ops + with _lock: + _image_ops += 1 + _image_ops_by_name[name] = _image_ops_by_name.get(name, 0) + 1 + + +def end_image_op(name: str = "image") -> None: + global _image_ops + with _lock: + _image_ops = max(0, _image_ops - 1) + if name in _image_ops_by_name: + _image_ops_by_name[name] -= 1 + if _image_ops_by_name[name] <= 0: + _image_ops_by_name.pop(name, None) + + +@contextmanager +def image_op(name: str = "image"): + begin_image_op(name) + try: + yield + finally: + end_image_op(name) + + +def activity() -> Dict[str, object]: + with _lock: + return { + "image_ops": _image_ops, + "image_ops_by_name": dict(_image_ops_by_name), + "uptime_s": round(time.time() - _started_at, 1), + } + + +def heavy_image_ops_active() -> int: + with _lock: + return _image_ops + + +def busy() -> bool: + return heavy_image_ops_active() > 0 diff --git a/src/agent_evidence.py b/src/agent_evidence.py index b3c7e2b22..e012ebe66 100644 --- a/src/agent_evidence.py +++ b/src/agent_evidence.py @@ -240,11 +240,13 @@ def _artifact_match_is_callable(instruction: str, match: re.Match[str], path: st def _artifact_match_is_email_host(instruction: str, match: re.Match[str]) -> bool: - """Reject the domain portion of an email address as an output path.""" + """Reject either dotted portion of an email address as an output path.""" start = match.start("path") prefix = instruction[max(0, start - 80):start] - return bool(re.search(r"[A-Za-z0-9_.+-]+@$", prefix)) + return instruction[match.end("path"):].startswith("@") or bool( + re.search(r"[A-Za-z0-9_.+-]+@$", prefix) + ) def _workspace_path_identity(value: str) -> str: diff --git a/src/agent_loop.py b/src/agent_loop.py index 257cc441d..3b45c77d5 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -162,7 +162,7 @@ def _resolved_tool_call_id( logger = logging.getLogger(__name__) -_MODEL_TOOL_SURFACES = {"none", "compact", "full"} +_MODEL_TOOL_SURFACES = {"none", "compact", "full", "spec"} _ROUTE_THINKING_MODES = {"auto", "on", "off"} _NO_THINKING_COMPACT_DOMAINS = { "email", @@ -2834,7 +2834,7 @@ def _parse_qwen_explicit_admin_request(text: str) -> Optional[tuple[str, str]]: if re.search(r"\b(?:disabled\s+tools?|tools?\s+(?:are\s+)?(?:currently\s+)?disabled)\b", q): return "manage_settings", json.dumps({"action": "list_tools"}) if re.search(r"\b(?:turn|enable|re-enable|reenable)\b.{0,40}\b(?:image\s+generation|images?|generate_image)\b", q): - return "manage_settings", json.dumps({"action": "enable_tool", "tool": "images"}) + return "manage_settings", json.dumps({"action": "list_tools"}) if re.search(r"\b(?:disable|turn\s+off)\b.{0,40}\b(?:image\s+generation|images?|generate_image)\b", q): return "manage_settings", json.dumps({"action": "disable_tool", "tool": "images"}) if listish and re.search(r"\bapi\s+tokens?\b|\btokens?\b", q): @@ -8057,7 +8057,7 @@ Generate an image. Line 1 = description, line 2 = model name, line 3 = WxH (e.g. "manage_tokens": "- ```manage_tokens``` — Generate or revoke API access tokens for external integrations. Args (JSON): {\"action\": \"list|create|delete\", ...}", "manage_documents": "- ```manage_documents``` — List, read/open, delete, or tidy documents in the editor panel. Args (JSON): {\"action\": \"list|read|delete|tidy\", ...}. `list` returns rows like `[Title](#document-) — lang, size, updated 5m ago` sorted MOST-RECENT FIRST; the user clicks the anchor to open. `read` (aliases: view/open/get) takes `document_id` and returns the content. When the user asks \"open/show/read my notes\" or \"what documents do I have\", use this — do NOT shell out, do NOT curl.", "manage_research": "- ```manage_research``` — List, read/open, or delete saved DEEP RESEARCH results from the Library. Args (JSON): {\"action\": \"list|read|delete\", \"id\": \"\", \"search\": \"...\"}. `list` returns rows like `[query](#research-) — N sources` MOST-RECENT FIRST; the user clicks to open. `read` (aliases: open/view/get) takes `id` and returns the report text + sources. Use when the user says \"open/read/find/delete my research\" or \"that report\". This IS how you read a finished report: when the user refers to a just-completed deep-research job (\"check it out\", \"read that report\", \"summarize the research\") WITHOUT giving an id, call `manage_research` with `action:list` to get the most-recent id, then `action:read` with that id, and answer from the returned text. Do NOT `web_fetch`/`app_api` the `/api/research/report/{id}` URL — that endpoint renders HTML for the browser, not clean text — and do NOT start a fresh `web_search`/`trigger_research` just to read an existing report. To START new research, use trigger_research instead.", - "manage_settings": "- ```manage_settings``` — View/change the REAL app settings (same ones the Settings panel writes) AND turn tools on/off. Change a setting: `{\"action\":\"set\",\"key\":\"...\",\"value\":\"...\"}` — keys accept friendly aliases, e.g. voice→tts_voice, \"search engine\"→search_provider, \"default model\"→default_model, \"teacher model\"→teacher_model, \"task/background model\"→task_model, \"image quality\"→image_quality, \"reminder channel\"→reminder_channel (browser|email|ntfy), \"agent timeout\"/\"max tool calls\"/\"token budget\". Read: `{\"action\":\"get\",\"key\":\"...\"}`; see all: `{\"action\":\"list\"}`; reset one: `{\"action\":\"reset\",\"key\":\"...\"}`. Use this when the user asks to change ANY preference instead of making them open Settings. Secrets/API keys are read-only (tell them to set those in the panel). Tool toggles: `{\"action\":\"disable_tool|enable_tool\",\"tool\":\"shell\"}` (aliases: shell/search/browser/documents/memory/skills/images/tasks/notes/calendar/email), list disabled: `{\"action\":\"list_tools\"}`.", + "manage_settings": "- ```manage_settings``` — View/change the REAL app settings (same ones the Settings panel writes) AND turn tools on/off. Change a setting: `{\"action\":\"set\",\"key\":\"...\",\"value\":\"...\"}` — keys accept friendly aliases, e.g. voice→tts_voice, \"search engine\"→search_provider, \"default model\"→default_model, \"teacher model\"→teacher_model, \"task/background model\"→task_model, \"image quality\"→image_quality, \"reminder channel\"→reminder_channel (browser|email|ntfy), \"agent timeout\"/\"max tool calls\"/\"token budget\". Read: `{\"action\":\"get\",\"key\":\"...\"}`; see all: `{\"action\":\"list\"}`; reset one: `{\"action\":\"reset\",\"key\":\"...\"}`. Use this when the user asks to change ANY preference instead of making them open Settings. Secrets/API keys are read-only (tell them to set those in the panel). Tool toggles: `{\"action\":\"disable_tool\",\"tool\":\"shell\"}` (aliases: shell/search/browser/documents/memory/skills/images/tasks/notes/calendar/email), list disabled: `{\"action\":\"list_tools\"}`. You cannot re-enable a disabled tool: that is done by the user in Settings > Agent Tools.", "manage_notes": """\ ```manage_notes {"action": "add", "title": "", "due_date": ""} @@ -8472,7 +8472,7 @@ def _agent_route_tool_mode( return is_api_model, is_ollama_native, ollama_openai_compat -def _configured_model_tool_surface( +def _configured_model_tool_surface_raw( endpoint_url: str, model: str, owner: Optional[str] = None, @@ -8561,6 +8561,19 @@ _ADMIN_KEYWORDS = [ "note", "notes", "todo", "todos", "reminder", "reminders", ] + +def _configured_model_tool_surface( + endpoint_url: str, + model: str, + owner: Optional[str] = None, + headers: Optional[Dict] = None, + endpoint_id: Optional[str] = None, +) -> str: + """Tool surface as the harness sees it: Ajax Spec runs the compact runtime + (same dispatch, prompts, completions); only Spec hooks read the raw mode.""" + value = _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers, endpoint_id=endpoint_id) + return "compact" if value == "spec" else value + def _detect_admin_intent(messages: List[Dict]) -> bool: """Check if the last user message suggests admin/management tool usage.""" for msg in reversed(messages): @@ -13565,6 +13578,20 @@ def _ody_qwen_terminal_tool_summary(tool_event: dict[str, Any], user_text: str = """Return a deterministic user-facing answer for tools we can render safely.""" tool_name = _resolved_tool_event_name(tool_event) output = str(tool_event.get("output") or "") + if tool_name in { + "mcp__email__list_email_accounts", "mcp__email__list_emails", + "mcp__email__search_emails", "mcp__email__read_email", + } and output.lstrip().startswith("{"): + # The external email bridge wraps stdout in a JSON result. Render its + # successful read payload, never the wrapper or a failed result. + try: + email_result = json.loads(output) + except (TypeError, ValueError): + email_result = None + if isinstance(email_result, dict) and isinstance(email_result.get("stdout"), str): + if email_result.get("exit_code") not in (None, 0) or email_result.get("error"): + return "" + output = email_result["stdout"] command = str(tool_event.get("command") or "") action = "" try: @@ -21019,9 +21046,179 @@ def _blocks_before_inference(turn_contract) -> bool: ) +def _spec_unavailable_reply(unavailable) -> str: + blocked = set(unavailable or ()) + if blocked & (set(WEB_TOOL_NAMES) | {"private_browser", "youtube_tool"}): + return ("Web or browser access is unavailable for this chat. Enable Web search " + "in the chat controls, if available, and ask me to check again. " + "I have not opened the page or verified its contents.") + return ("The tool needed for this request is unavailable in this chat. " + "Enable the required capability, if available, and try again. " + "I have not performed the requested lookup or action.") + + from src.agent_runtime.authority import MISSING_AUTHORITY, active_request_authority, with_request_authority + +def _apply_spec_decision_to_contract(turn_contract, picks, decided, disabled_tools, tool_policy): + from src.turn_contract import _ACTIVE_CONTRACT as _spec_active_contract + disabled = set(disabled_tools or ()) + from src.spec_router import latest_decision + _spec_decision = latest_decision() or {} + _spec_choice = _spec_decision.get("choice") + _spec_direct = (os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" + and decided and _spec_choice == "answer_directly") + try: + _mgr = get_mcp_manager() + mcp_schemas = _mgr.get_all_openai_schemas(_load_mcp_disabled_map()) if _mgr else [] + except Exception: + mcp_schemas = [] + mcp_names = {(sc.get("function") or {}).get("name") for sc in mcp_schemas} + if _spec_choice == "ask_user": + # ask_user = a tool is needed but a detail may be missing: offer Spec's top tool too, so the model + # can act when the detail is there or ask when it is not. + _top = _spec_decision.get("top") or [] + if _top: + picks = set(picks) | {_top[0][0]} + picks = {(f"mcp__email__{n}" if f"mcp__email__{n}" in mcp_names else n) for n in picks} + strict = os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" + base_offered = set(turn_contract.offered) + # The router sees the full catalog, while this request's policy may + # deny a tool (for example, a browser turn with Web access switched + # off). Preserve that conflict as an unavailable contract. Keeping the + # old empty menu instead lets the model invent a tool result in prose. + _top = _spec_decision.get("top") or [] + _primary = str(_top[0][0]) if _spec_choice and str(_spec_choice).startswith("t") and _top else "" + if _primary and f"mcp__email__{_primary}" in mcp_names: + _primary = f"mcp__email__{_primary}" + # A policy denial is surfaced even if the router chose outside the + # base inventory; it never grants access to that tool. + def _denied(name): + return (name in disabled + or (tool_policy is not None and ( + tool_policy.blocks(name) + or (tool_policy.disable_mcp and name.startswith("mcp__"))))) + _unavailable_pick = "" + if _primary in picks and _denied(_primary): + _unavailable_pick = _primary + elif picks and all(_denied(name) for name in picks): + _unavailable_pick = sorted(picks)[0] + if _unavailable_pick: + blocked = replace( + turn_contract, required=frozenset(), offered=frozenset(), + unavailable=frozenset({_unavailable_pick}), schema_json=(), + required_read_operation=None, routing_experiment="spec-policy-denied", + ) + _spec_active_contract.set(blocked) + logger.info("[spec-harness] router pick %s unavailable under request policy", _unavailable_pick) + return blocked + want = set(picks) - disabled + # Always-on basics only when the base turn offers them, so the Spec + # arm never offers a tool the base arm would not. + if not _spec_direct: + want |= {"ask_user", "update_plan"} & base_offered + if not (strict and decided): + want |= base_offered + want &= base_offered + if not want: + logger.info("[spec-harness] router picks %s outside base offered %s; base contract kept", + sorted(picks), sorted(base_offered)) + return turn_contract + offer = want & base_offered & set(turn_contract.executable) + # A router may narrow the menu only when every existing obligation + # remains satisfiable. It cannot redefine the user's requested work. + required_read = getattr(turn_contract, "required_read_operation", None) + if (not set(turn_contract.required) <= offer or turn_contract.unavailable + or (required_read is not None and required_read.tool not in offer)): + return turn_contract + by_name = {} + for sc in list(FUNCTION_TOOL_SCHEMAS) + list(mcp_schemas): + nm = (sc.get("function") or {}).get("name") + if nm in offer and nm not in by_name: + by_name[nm] = sc + for sc in turn_contract.schemas(): + nm = (sc.get("function") or {}).get("name") + if nm in offer: + by_name[nm] = sc + if not by_name and not _spec_direct: + logger.info("[spec-harness] router picks %s not executable this turn; contract kept", sorted(picks)) + return turn_contract + # Required tools and the required read carry over unchanged: harness + # completion logic keys on them (e.g. a calendar list is a read, not an + # effect that may end with "Done."). If one lost its schema, contract + # validation fails and the original contract is retained below. + new = replace( + turn_contract, offered=frozenset(by_name), + schema_json=tuple(json.dumps(by_name[n], ensure_ascii=False, sort_keys=True) for n in sorted(by_name)), + routing_experiment="spec-router") + _spec_active_contract.set(new) + logger.info("[spec-harness] contract replaced by router: offered=%s (was %s, decided=%s)", + sorted(by_name), sorted(turn_contract.offered), decided) + return new + + +def _spec_router_contract(turn_contract, messages, disabled_tools, endpoint_url, model, owner, headers, + tool_policy=None): + """Ajax Spec: return a copy of the turn contract whose offered tools are the + router's picks (plus ask_user/update_plan), or the contract unchanged. + + Runs before any runtime dispatch so the compact preview runtime and the + main loop both serve the router's set; only the offered tools change. + """ + # Contexts can be inherited by a child task; a turn must never inherit a + # previous turn's decision when routing is disabled or unavailable. + from src import spec_router as _router_context + if hasattr(_router_context, "clear_decision"): + _router_context.clear_decision() + if (turn_contract is None + or getattr(turn_contract, "routing_experiment", "") in {"spec-router", "spec-policy-denied"} + or (tool_policy is not None and tool_policy.block_all_tool_calls)): + return turn_contract + if os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() not in {"strict", "additive"}: + return turn_contract + try: + if _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers) != "spec": + return turn_contract + except Exception: + return turn_contract + try: + from src.spec_router import spec_route_tools as _spec_route_tools_early + picks, decided = _spec_route_tools_early( + _extract_last_user_message(messages), FUNCTION_TOOL_SCHEMAS, set(), messages, return_decision=True) + return _apply_spec_decision_to_contract(turn_contract, picks, decided, disabled_tools, tool_policy) + except Exception as exc: + logger.warning("[spec-harness] router failed; retaining original contract: %s", exc) + return turn_contract + + +async def _spec_router_contract_async(turn_contract, messages, disabled_tools, endpoint_url, model, owner, headers, + tool_policy=None): + """Asynchronous non-blocking turn contract routing.""" + from src import spec_router as _router_context + if hasattr(_router_context, "clear_decision"): + _router_context.clear_decision() + if (turn_contract is None + or getattr(turn_contract, "routing_experiment", "") in {"spec-router", "spec-policy-denied"} + or (tool_policy is not None and tool_policy.block_all_tool_calls)): + return turn_contract + if os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() not in {"strict", "additive"}: + return turn_contract + try: + if _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers) != "spec": + return turn_contract + except Exception: + return turn_contract + try: + from src.spec_router import spec_route_tools_async as _spec_route_tools_early_async + picks, decided = await _spec_route_tools_early_async( + _extract_last_user_message(messages), FUNCTION_TOOL_SCHEMAS, set(), messages, return_decision=True) + return _apply_spec_decision_to_contract(turn_contract, picks, decided, disabled_tools, tool_policy) + except Exception as exc: + logger.warning("[spec-harness] router failed; retaining original contract: %s", exc) + return turn_contract + + @with_request_authority @with_turn_contract @with_teacher_takeover @@ -21099,6 +21296,41 @@ async def stream_agent_loop( set(tool_policy.all_disabled_names()) if tool_policy else set() )) + # Ajax Spec driver: the Spec router picks every step (src/spec_driver.py). + if (os.getenv("ODYSSEUS_SPEC_MODE", "").strip().lower() == "driver" + and not (tool_policy and tool_policy.block_all_tool_calls) + and _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers) == "spec"): + yield 'data: ' + json.dumps({"delta": "Spec driver mode is unavailable because its authority and completion checks are not supported. No action was performed."}) + '\n\n' + yield 'data: [DONE]\n\n' + return + + turn_contract = await _spec_router_contract_async( + turn_contract, messages, disabled_tools, endpoint_url, model, owner, headers, + tool_policy=tool_policy) + if (turn_contract is not None + and getattr(turn_contract, "routing_experiment", "") == "spec-policy-denied"): + # A selected but disallowed tool is a policy outcome, not an invitation + # to generate an ungrounded answer with an empty tool menu. + reply = _spec_unavailable_reply(turn_contract.unavailable) + yield 'data: ' + json.dumps({"type": "turn_contract", **turn_contract.audit()}) + '\n\n' + yield 'data: ' + json.dumps({"delta": reply}) + '\n\n' + yield 'data: [DONE]\n\n' + return + + try: + _spec_turn = _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers) == "spec" + except Exception: + _spec_turn = False + # Without a turn contract (scheduled, skill and monitor runs) no Spec + # routing applies; the turn runs on the compact surface as usual. + if _spec_turn and _blocks_before_inference(turn_contract): + unavailable = ", ".join(sorted(turn_contract.unavailable)) + yield 'data: ' + json.dumps({"type": "turn_contract", **turn_contract.audit()}) + '\n\n' + yield 'data: ' + json.dumps({"delta": "I can’t perform this request with the currently permitted tools " + f"(unavailable: {unavailable}). I haven’t substituted another tool."}) + '\n\n' + yield 'data: [DONE]\n\n' + return + if is_compact_preview_contract(turn_contract): from src.clean_agent_preview import stream_preview async for chunk in stream_preview( @@ -24126,6 +24358,15 @@ async def stream_agent_loop( headers=candidate_headers, endpoint_id=(route_descriptor or {}).get("endpoint_id"), ) + # "spec" = compact tools plus a small-model router; the harness-facing + # surface already reads "compact", so detect spec from the raw mode. + try: + _spec_surface = getattr(turn_contract, "routing_experiment", "") == "spec-router" and _configured_model_tool_surface_raw( + candidate_url, candidate_model, owner, headers=candidate_headers, + endpoint_id=(route_descriptor or {}).get("endpoint_id")) == "spec" + except Exception: + _spec_surface = False + _spec_offered = None textual_tools = ( force_textual_tool_transport or force_textual_tools @@ -24159,6 +24400,57 @@ async def stream_agent_loop( _last_user, _intent_domains, ) + if _spec_surface: + from src.spec_router import spec_route_tools as _spec_route_tools + + if turn_contract is not None and getattr(turn_contract, "routing_experiment", "") == "spec-router": + _spec_added, _spec_decided = set(turn_contract.offered) - {"ask_user", "update_plan"}, True + else: + _spec_added, _spec_decided = _spec_route_tools( + _last_user, + FUNCTION_TOOL_SCHEMAS, + disabled_tools, + compacted_source, + return_decision=True, + ) + _spec_added = _spec_added - set(disabled_tools) + # The router catalog uses the legacy native email names; the + # live tools are the MCP ones, and the legacy alias schemas are + # dropped when the MCP server is connected. Map picks over. + try: + from src.tool_utils import get_mcp_manager as _spec_mcp_mgr + _mgr = _spec_mcp_mgr() + _mcp_names = { + (schema.get("function") or {}).get("name") + for schema in (_mgr.get_all_openai_schemas(_load_mcp_disabled_map()) if _mgr else []) + } + except Exception: + _mcp_names = set() + _spec_added = { + (f"mcp__email__{name}" if f"mcp__email__{name}" in _mcp_names else name) + for name in _spec_added + } + _spec_mode = os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() + _compact_before = set(route_tools) + if _spec_mode == "strict" and _spec_decided: + # The router decides which tools are injected: its pick(s), + # tools already used in this chat, and the always-on basics. + _spec_strict = (_spec_added | {"ask_user", "update_plan"}) & set(turn_contract.offered) - set(disabled_tools) + if _spec_strict: + route_tools = _spec_strict + _spec_offered = frozenset(route_tools) + else: + logger.info("[spec-harness] strict set empty after disabled-tool filter; keeping compact set") + route_tools = set(route_tools) | _spec_added + else: + route_tools = set(route_tools) | _spec_added + logger.info("[spec-debug] disabled_tools (%d): %s | policy block_all=%s disable_mcp=%s", + len(set(disabled_tools)), sorted(set(disabled_tools))[:80], + getattr(tool_policy, "block_all_tool_calls", None), getattr(tool_policy, "disable_mcp", None)) + logger.info( + "[spec-harness] mode=%s decided=%s compact=%s final=%s", + _spec_mode, _spec_decided, sorted(_compact_before), sorted(route_tools), + ) # The compact router only receives user text, while native task # inputs may be declared out-of-band by the runner. Reapply that # concrete input contract after compaction so an implicit video @@ -24199,7 +24491,7 @@ async def stream_agent_loop( _irrelevant_local_media_web_tools.add("private_browser") route_tools.difference_update(_irrelevant_local_media_web_tools) if turn_contract is not None and tool_surface != "full": - route_tools = set(turn_contract.offered) + route_tools = set(_spec_offered) if _spec_offered else set(turn_contract.offered) if prompt_compact: route_tools.update( _blocked_network_recovery_tools(tool_events) @@ -24299,7 +24591,7 @@ async def stream_agent_loop( disabled_tools, needs_admin=_needs_admin, relevant_tools=prompt_route_tools, - preserve_conversation=turn_contract is not None, + preserve_conversation=(turn_contract is not None) or _spec_surface, mcp_disabled_map=_mcp_disabled_map, compact=prompt_compact, owner=owner, @@ -24511,6 +24803,13 @@ async def stream_agent_loop( _prepend_agent_directive(route_messages, build_active_plan_note(approved_plan)) if guide_only: _prepend_agent_directive(route_messages, GUIDE_ONLY_DIRECTIVE) + if _spec_offered and os.getenv("ODYSSEUS_SPEC_TRIM_PROMPT", "1") != "0": + from src.spec_router import trim_system_prompt as _spec_trim + + route_messages = _spec_trim(route_messages, _spec_offered) + from src.spec_router import drop_skills_index as _spec_drop_skills + + route_messages = _spec_drop_skills(route_messages, _spec_offered) return { "messages": route_messages, "mcp_schemas": route_mcp_schemas, @@ -24519,6 +24818,8 @@ async def stream_agent_loop( "is_ollama_native": is_native_ollama, "ollama_openai_compat": is_ollama_compat, "tool_surface": tool_surface, + "spec_offered": _spec_offered, + "spec_surface": _spec_surface, "ody_qwen_finetune_model": is_ody, "qwen38_tool_router": _is_qwen38_tool_router(candidate_model) and not _full_inventory_mode, "ody_doc_finetune_mode": doc_mode, @@ -25076,6 +25377,15 @@ async def stream_agent_loop( if (schema.get("function", {}).get("name") or schema.get("name")) not in disabled_tools ] + _spec_names = route_state.get("spec_offered") + if _spec_names: + # Ajax Spec: the router, not the turn contract, decides which + # tools are injected for this turn. + contract_schemas = [ + schema for schema in contract_schemas + if (schema.get("function", {}).get("name") or schema.get("name")) + in _spec_names + ] return _filter_route_tool_schemas(_apply_tool_surface_to_schemas(contract_schemas, tool_surface)) if route_state["is_api_model"]: if tool_surface == "full": @@ -25137,15 +25447,18 @@ async def stream_agent_loop( and schema.get("name") not in disabled_tools ] if _pure_web_turn and tool_surface != "full": - allowed = _web_only_route_tools(_last_user, disabled_tools) - schemas = [ - schema for schema in schemas - if ( - schema.get("function", {}).get("name") - or schema.get("name") - ) in allowed - or schema.get("function", {}).get("name") in external_names - ] + strict_spec = (route_state.get("spec_surface") and + os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict") + if not strict_spec: + allowed = _web_only_route_tools(_last_user, disabled_tools) + schemas = [ + schema for schema in schemas + if ( + schema.get("function", {}).get("name") + or schema.get("name") + ) in allowed + or schema.get("function", {}).get("name") in external_names + ] schemas = _drop_legacy_email_alias_schemas_when_mcp_available(schemas) schemas = _apply_tool_surface_to_schemas(schemas, tool_surface) if ( @@ -25997,6 +26310,8 @@ async def stream_agent_loop( "is_ollama_native": _is_ollama_native, "ollama_openai_compat": _ollama_openai_compat, "tool_surface": _route_state.get("tool_surface", ""), + "spec_offered": _route_state.get("spec_offered"), + "spec_surface": _route_state.get("spec_surface"), "ody_qwen_finetune_model": _ody_qwen_finetune_model, "qwen38_tool_router": _qwen38_tool_router, "ody_doc_finetune_mode": _ody_doc_finetune_mode, @@ -26012,10 +26327,19 @@ async def stream_agent_loop( _active_route_state["request_messages"] = _initial_route_request_messages all_tool_schemas = _tool_schemas_for_route(_active_route_state) if turn_contract is not None: - _relevant_tools = set(turn_contract.offered) + _relevant_tools = ( + set(_route_state.get("spec_offered")) + if _route_state.get("spec_offered") + else set(turn_contract.offered) + ) agent_stream_timeout = int(get_setting("agent_stream_timeout_seconds", 300) or 300) _tool_names_sent = [t.get("function", {}).get("name") for t in (all_tool_schemas or []) if t.get("function")] + if round_num == 1: + try: + logger.info("[spec-debug] prompt composition (chars): %s", [(m.get("role"), len(str(m.get("content") or ""))) for m in messages]) + except Exception: + pass logger.info(f"[agent-debug] round={round_num} model={model} _is_api_model={_is_api_model} tools_sent={len(_tool_names_sent)} tool_names={_tool_names_sent} relevant_tools={sorted(_relevant_tools)[:50] if _relevant_tools else 'ALL'}") _routing_intentional_exclusions = set(disabled_tools) if ( @@ -27605,7 +27929,11 @@ async def stream_agent_loop( ), active_document=active_document, last_user=_last_user, - offered_tool_names=(set(turn_contract.offered) if turn_contract is not None else set(_tool_names_sent)), + offered_tool_names=( + set(turn_contract.offered) + if turn_contract is not None and not _active_route_state.get("spec_offered") + else set(_tool_names_sent) + ), recover_unoffered_tool_names=_recover_unoffered_local_tools, passthrough_tool_names={ schema["function"]["name"] @@ -32169,6 +32497,29 @@ async def stream_agent_loop( budget_hit = False local_network_budget_hit = False local_inspection_budget_hit = False + if _active_route_state.get("spec_offered") and os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" and tool_blocks and not used_native and _is_api_model: + # Ajax Spec with a native-API model: a round without a native call + # carries only harness-synthesized recovery blocks. Drop them all. + logger.info("[spec-harness] dropped %d synthesized tool block(s) (no native call): %s", + len(tool_blocks), [b.tool_type for b in tool_blocks]) + tool_blocks = []; converted_calls = [] + if ( + _active_route_state.get("spec_offered") and os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" and used_native and tool_blocks + and len(converted_calls) == len(tool_blocks) + ): + # Ajax Spec: only the model's own native calls run; harness- + # synthesized recovery blocks (no native call) are dropped. When + # converted_calls was reset (singleton collapse) keep every block. + _spec_keep = [ + i for i in range(len(tool_blocks)) + if i < len(converted_calls) and converted_calls[i] is not None + ] + if len(_spec_keep) != len(tool_blocks): + logger.info("[spec-harness] dropped %d synthesized tool block(s): %s", + len(tool_blocks) - len(_spec_keep), + [tool_blocks[i].tool_type for i in range(len(tool_blocks)) if i not in _spec_keep]) + tool_blocks = [tool_blocks[i] for i in _spec_keep] + converted_calls = [converted_calls[i] for i in _spec_keep] for i, block in enumerate(tool_blocks): native_call = converted_calls[i] if i < len(converted_calls) else None tool_call_id = _resolved_tool_call_id( @@ -33463,7 +33814,8 @@ async def stream_agent_loop( ), } if ( - (_qwen38_tool_router or _full_inventory_mode) and (_pure_web_turn or _contextual_public_web_followup) + not (_active_route_state.get("spec_offered") and os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" and block.tool_type == "private_browser") + and (_qwen38_tool_router or _full_inventory_mode) and (_pure_web_turn or _contextual_public_web_followup) and not _artifact_creation_requested and (block.tool_type == "web_search" or _web_execution_budget.searches) and not _web_execution_budget.admit( diff --git a/src/agent_runtime/effect_adapters.py b/src/agent_runtime/effect_adapters.py index 0e1defd2b..34c3772cb 100644 --- a/src/agent_runtime/effect_adapters.py +++ b/src/agent_runtime/effect_adapters.py @@ -230,6 +230,18 @@ def classify(capture: DispatchCapture) -> dict[str, Any] | None: if capture.backend is not None: from src.agent_runtime.resources import ExternalResource if isinstance(capture.backend.resource, ExternalResource): + # A read through an external backend is still a read. The tool + # capability registry is sealed by the dispatcher, and unknown or + # mixed-effect calls continue to receive an external effect claim. + if all(bound is None for bound in (capture.browser, capture.filesystem, + capture.process, capture.owned)): + from src.tool_capabilities import ToolEffect, capabilities_for_action + capabilities = capabilities_for_action(capture.backend.transport_tool, + capture.backend.exact_input) + reads = frozenset({ToolEffect.READ_PUBLIC, ToolEffect.READ_WORKSPACE, + ToolEffect.READ_PRIVATE, ToolEffect.BROKERED_NETWORK_READ}) + if capabilities.known and capabilities.effects and capabilities.effects <= reads: + return None external = True impact = (*impact, resource_ref(capture.backend.resource, "backend")) return {"impact_scope": impact, "dependencies": dependencies, "obligations": obligations, "external": external} diff --git a/src/agent_tools/__init__.py b/src/agent_tools/__init__.py index 7db8cffbb..7689f6ead 100644 --- a/src/agent_tools/__init__.py +++ b/src/agent_tools/__init__.py @@ -29,6 +29,7 @@ from .interaction_tools import AskUserTool, UpdatePlanTool from .model_interaction_tools import ChatWithModelTool, AskTeacherTool, ListModelsTool from .bg_job_tools import ManageBgJobsTool from .session_tools import CreateSessionTool, ListSessionsTool, SendToSessionTool, ManageSessionTool +from .editor_tools import PlanEditTool, RunEditorCommandTool, GetEditorScreenshotTool, AiEditImageTool from .admin_tools import ( ADMIN_TOOL_HANDLERS, do_manage_endpoints, do_manage_mcp, do_manage_webhooks, @@ -72,6 +73,10 @@ TOOL_HANDLERS = { "list_sessions": ListSessionsTool().execute, "send_to_session": SendToSessionTool().execute, "manage_session": ManageSessionTool().execute, + "plan_edit": PlanEditTool().execute, + "run_editor_command": RunEditorCommandTool().execute, + "get_editor_screenshot": GetEditorScreenshotTool().execute, + "ai_edit_image": AiEditImageTool().execute, } # Config/integration admin tools (manage_endpoints/mcp/webhooks/tokens/settings). TOOL_HANDLERS.update(ADMIN_TOOL_HANDLERS) diff --git a/src/agent_tools/admin_tools.py b/src/agent_tools/admin_tools.py index f0a7662c2..419b7f742 100644 --- a/src/agent_tools/admin_tools.py +++ b/src/agent_tools/admin_tools.py @@ -585,7 +585,9 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict: return _ALIASES_SET.get(k2, (k or "").strip()) def _is_managed_key(key): - return key in DEFAULT_SETTINGS and key not in RETIRED_SETTING_KEYS + # Tool policy is not a general setting, even if a future default + # is added: resetting it must never re-enable disabled tools. + return key != "disabled_tools" and key in DEFAULT_SETTINGS and key not in RETIRED_SETTING_KEYS _ENUMS = { "image_quality": ["low", "medium", "high"], @@ -713,13 +715,35 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict: save_settings(s) return {"response": f"Reset {key} to default ({DEFAULT_SETTINGS[key]}).", "exit_code": 0} - elif action in ("disable_tool", "enable_tool", "list_tools"): + elif action == "enable_tool": + # The global disabled_tools denylist is the one hard permission + # boundary the Settings -> Agent Tools panel offers admins. It + # must only ever be *narrowed* from inside an agent turn: a + # model that can call enable_tool can undo the boundary itself, + # and any web page, email, or document the agent reads can ask + # it to (prompt injection re-enabling bash/email/etc.). Widening + # the tool set stays a human action in the Settings panel. + tool_name = (args.get("tool") or args.get("name") or "").strip().lower() + return { + "error": ( + f"Re-enabling tools is not available from chat" + f"{f' (requested: {tool_name})' if tool_name else ''}. " + "Disabled tools are an admin policy: ask the user to turn the tool " + "back on in Settings > Agent Tools. You can still disable tools " + "(disable_tool) or list the current state (list_tools)." + ), + "exit_code": 1, + } + + elif action in ("disable_tool", "list_tools"): # Tool-toggle actions. These edit settings.json:disabled_tools # (the global list read on every chat request) rather than # prefs.json. Friendly aliases accepted: "shell" -> "bash", # "search" -> "web_search", "browser" -> browser tools, # "documents" -> the document tool set, "memory" -> - # manage_memory, etc. + # manage_memory, etc. Only disable_tool mutates: see the + # enable_tool branch above for why the agent can never widen + # the denylist. from src.settings import get_setting, save_settings, load_settings _ALIASES = { "shell": ["bash"], @@ -753,7 +777,8 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict: "response": ( f"Currently disabled: {', '.join(current) if current else '(none)'}.\n" "Common toggles: shell (bash), search (web_search), browser, documents, " - "memory, skills, images, tasks, notes, calendar, email." + "memory, skills, images, tasks, notes, calendar, email. " + "To re-enable a tool, open Settings > Agent Tools." ), "disabled": list(current), "exit_code": 0, @@ -767,22 +792,19 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict: settings = load_settings() current = list(settings.get("disabled_tools") or []) before = set(current) - if action == "disable_tool": - for t in targets: - if t not in current: - current.append(t) - else: # enable_tool - current = [t for t in current if t not in targets] + for t in targets: + if t not in current: + current.append(t) after = set(current) settings["disabled_tools"] = current save_settings(settings) - verb = "Disabled" if action == "disable_tool" else "Enabled" changed = sorted(after.symmetric_difference(before)) return { "response": ( - f"{verb} {tool_name} ({', '.join(targets)}). " - f"Now disabled: {', '.join(current) if current else '(none)'}." + f"Disabled {tool_name} ({', '.join(targets)}). " + f"Now disabled: {', '.join(current) if current else '(none)'}. " + "Re-enabling is done by the user in Settings > Agent Tools." ), "changed": changed, "disabled": list(current), diff --git a/src/agent_tools/editor_tools.py b/src/agent_tools/editor_tools.py new file mode 100644 index 000000000..aec99a56a --- /dev/null +++ b/src/agent_tools/editor_tools.py @@ -0,0 +1,697 @@ +"""In-process agent tools for the scoped ``photo_editor`` capability family. + +These handlers run as the logged-in owner and drive the image editor over the +owner-scoped in-process bridge (``/api/editor/bridge/*``) plus the app's +existing image transforms (``/api/image/*``). They are only offered to the +model when the ``photo_editor`` family is active (the Quick Edit popup or an +explicit photo-edit chat turn), never in the general tool list. + +Loopback calls carry the internal token and impersonate ``owner`` (middleware +app.py), so the editor bridge resolves the same ``owner:session`` scope the +browser subscriber uses. +""" +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import logging +import re +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional + +import httpx + +from src.tools._common import _INTERNAL_BASE, _internal_headers + +logger = logging.getLogger(__name__) + + +def _provider_message() -> str: + try: + from src.ai_interaction import IMAGE_PROVIDER_REQUIRED_MESSAGE + + return IMAGE_PROVIDER_REQUIRED_MESSAGE + except Exception: + return ( + "Removing/adding objects needs an image generation provider; enable one " + "in Settings > Models." + ) + + +def _provider_available(owner: Optional[str]) -> bool: + try: + from src.ai_interaction import image_editing_provider_available + + return bool(image_editing_provider_available(owner)) + except Exception: + return True + +# Local diffusion/image servers can take 90-110s per image (8 steps) and +# longer on a cold start; the previous 120s default timed out mid-generation. +_EDITOR_TIMEOUT = 300.0 +# A mask covering less than this fraction of the image cannot drive a useful +# regional inpaint (tiny figures, empty SAM output): fall back to grounding or +# whole-image instruction mode instead. +_MIN_MASK_FRACTION = 0.002 +_AI_OPS = { + "remove-bg": "/api/image/remove-bg", + "upscale": "/api/image/upscale-local", + "denoise": "/api/image/denoise", + "enhance-face": "/api/image/enhance-face", + "inpaint": "/api/image/inpaint", + "generative": "/api/image/inpaint", +} +_PLACE_OP = "layers.placeImage" + + +def _parse(content: Any) -> Dict[str, Any]: + raw = content if isinstance(content, str) else json.dumps(content or {}) + try: + parsed = json.loads(raw) if raw.strip() else {} + except (ValueError, TypeError): + parsed = {} + return parsed if isinstance(parsed, dict) else {} + + +def _session(args: Dict[str, Any], ctx: Dict[str, Any]) -> str: + # Explicit editor session bound in context is authoritative; + # otherwise fallback to caller/tool args, then default. + # Note: ctx.get("session_id") is the chat/turn UUID, NOT the editor canvas session. + return str(ctx.get("editor_session") or args.get("session") or args.get("editor_session") or "default") + + +def _incognito(ctx: Dict[str, Any]) -> bool: + runtime = ctx.get("client_runtime_context") + state = runtime.get("web_ui_state") if isinstance(runtime, dict) else None + return ctx.get("incognito") is True or (isinstance(state, dict) and state.get("incognito") is True) + + +async def _post(path: str, payload: Dict[str, Any], owner: Optional[str], timeout: float = _EDITOR_TIMEOUT) -> Dict[str, Any]: + headers = {**_internal_headers(owner), "Content-Type": "application/json"} + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post(f"{_INTERNAL_BASE}{path}", json=payload, headers=headers) + resp.raise_for_status() + try: + return resp.json() + except ValueError: + return {"raw": resp.text} + + +def _as_data_url(value: Any) -> Optional[str]: + if not isinstance(value, str) or not value: + return None + if value.startswith("data:"): + return value + if value.startswith("/"): + return None + return f"data:image/png;base64,{value}" + + +def _data_url_bytes(data_url: str) -> bytes: + _, _, b64 = data_url.partition(",") + return base64.b64decode(b64) + + +def _image_reference_path(value: str) -> bool: + # Never use the internal tool credential to fetch an arbitrary app route. + generated = re.fullmatch(r"/api/generated-image/[a-f0-9]{8,64}\.(?:png|jpg|jpeg|webp|gif)", value) + upload = re.fullmatch(r"/api/upload/[a-fA-F0-9]{32}(?:\.[A-Za-z0-9]+)?", value) + return bool(generated or upload) + + +def _persist_png(image_bytes: bytes, owner: Optional[str], session_id: Optional[str], prompt: str, model: str) -> Dict[str, Any]: + """Save an edited/generated PNG and return {image_id, image_url, image_size}.""" + from core.database import GalleryImage, SessionLocal + from src.constants import GENERATED_IMAGES_DIR + + db = SessionLocal() + image_path = None + try: + root = Path(GENERATED_IMAGES_DIR).resolve() + root.mkdir(parents=True, exist_ok=True) + filename = f"{uuid.uuid4().hex[:12]}.png" + image_path = root / filename + image_path.write_bytes(image_bytes) + new_id = str(uuid.uuid4()) + width = height = 0 + try: + from PIL import Image + + with Image.open(io.BytesIO(image_bytes)) as img: + width, height = img.size + except Exception: + pass + size = f"{width}x{height}" if width and height else "" + # The editor bridge uses an owner-scoped session id (e.g. "pewds:3") + # that is not a chat-sessions row. GalleryImage.session_id is a FK to + # sessions.id, so drop an unknown id instead of raising IntegrityError. + if session_id: + from core.database import Session as _ChatSession + + try: + known = db.query(_ChatSession.id).filter(_ChatSession.id == session_id).first() + except Exception: + known = None + if known is None: + session_id = None + derived = GalleryImage( + id=new_id, + filename=filename, + prompt=prompt or None, + caption=None, + model=model, + size=size or None, + quality=None, + tags=None, + ai_tags=None, + session_id=session_id, + album_id=None, + owner=owner, + file_hash=hashlib.sha256(image_bytes).hexdigest(), + file_size=len(image_bytes), + width=width or None, + height=height or None, + ) + db.add(derived) + db.commit() + return {"image_id": new_id, "image_url": f"/api/generated-image/{filename}", "image_size": size or None} + except Exception: + db.rollback() + if image_path is not None: + try: + image_path.unlink(missing_ok=True) + except OSError: + logger.warning("Failed to remove uncommitted editor image", exc_info=True) + raise + finally: + db.close() + + +async def _place_layer(data_url: str, name: str, owner: Optional[str], session: str, *, masked: bool = False, blend: str = "normal", opacity: float = 100.0, mode: Optional[str] = None) -> bool: + params: Dict[str, Any] = { + "dataUrl": data_url, + "name": name, + "masked": bool(masked), + "blend": blend or "normal", + "opacity": opacity, + } + # Forward the edit mode so the client can build a difference mask for + # instruction-mode results: the whole image was regenerated under an + # instruction prompt, so untouched pixels must stay pixel-identical (the + # client keeps only the changed region) instead of the new layer overwriting + # the entire canvas. Omit the keys entirely when no mode is given so the + # client keeps its current behavior (e.g. generate_image must not diff-mask). + if mode: + params["mode"] = mode + params["differenceMask"] = mode == "instruction" + if mode == "instruction": + # Exclude a thin edge band from the client difference mask: full-frame + # regenerations can shift pixels along the border. + params["edgeExcludePx"] = 3 + payload = { + "session": session, + "op": _PLACE_OP, + "params": params, + "wait": True, + "timeout": 60, + } + try: + data = await _post("/api/editor/bridge/command", payload, owner, timeout=90) + return bool(data.get("ok") is True and data.get("completed", True) is not False) + except Exception as exc: # pragma: no cover - defensive + logger.warning("editor layer placement failed: %s", exc) + return False + + +async def _fetch_selection_mask(owner: Optional[str], session: str) -> Optional[str]: + """Return the editor's current selection as a data-URL mask, or None. + + Generative ops (inpaint/generative) need a mask; the planner/judge may not + have supplied one, so fall back to whatever is selected in the editor. + """ + payload = { + "session": session, + "op": "selection.exportMask", + "params": {}, + "wait": True, + "timeout": 30, + } + try: + data = await _post("/api/editor/bridge/command", payload, owner, timeout=45) + except Exception as exc: # pragma: no cover - defensive + logger.warning("editor selection mask fetch failed: %s", exc) + return None + result = data.get("result") if isinstance(data, dict) else None + if isinstance(result, dict): + return _as_data_url(result.get("mask")) + return None + + +def _mask_white_fraction(data_url: Optional[str]) -> float: + """Fraction of a grayscale mask that is selected (white, >=128).""" + if not data_url: + return 0.0 + try: + from PIL import Image as _PILImage + + with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))).convert("L") as im: + total = im.width * im.height + if not total: + return 0.0 + hist = im.histogram() + return sum(hist[128:]) / total + except Exception: + return 0.0 + + +def _image_is_blank(data_url: str) -> bool: + """True when a result has an alpha channel that is essentially transparent.""" + try: + from PIL import Image as _PILImage + + with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))) as im: + if im.mode not in ("RGBA", "LA") and "transparency" not in im.info: + return False + alpha = im.convert("RGBA").getchannel("A") + hist = alpha.histogram() + total = sum(hist) + return total > 0 and (sum(hist[1:]) / total) < 0.001 + except Exception: + return False + + +def _rect_mask_data_url(image_data_url: str, box: List[float], *, pad_frac: float = 0.15) -> Optional[str]: + """Build a rectangular mask data URL from a [x1,y1,x2,y2] box, dilated. + + Used when the grounding detector finds an object but SAM returns an empty + mask (tiny figures): a dilated box guarantees a usable region. + """ + try: + from PIL import Image as _PILImage, ImageDraw + + with _PILImage.open(io.BytesIO(_data_url_bytes(image_data_url))) as im: + w, h = im.size + x1, y1, x2, y2 = (float(v) for v in box) + bw, bh = x2 - x1, y2 - y1 + x1, y1 = max(0.0, x1 - bw * pad_frac), max(0.0, y1 - bh * pad_frac) + x2, y2 = min(float(w), x2 + bw * pad_frac), min(float(h), y2 + bh * pad_frac) + mask = _PILImage.new("L", (w, h), 0) + ImageDraw.Draw(mask).rectangle([x1, y1, x2, y2], fill=255) + buf = io.BytesIO() + mask.save(buf, "PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() + except Exception: + return None + + +async def _ground_box(owner: Optional[str], image_data_url: str, label: str) -> Optional[List[float]]: + """Use the in-app grounding detector (OWL-ViT via /api/image/mask) for a box.""" + try: + data = await _post("/api/image/mask", {"image": image_data_url, "text": label}, owner, timeout=90) + except Exception as exc: # pragma: no cover - defensive + logger.warning("editor grounding box failed: %s", exc) + return None + box = data.get("bbox") if isinstance(data, dict) else None + if isinstance(box, (list, tuple)) and len(box) == 4: + try: + return [float(v) for v in box] + except Exception: + return None + return None + + +def _ensure_keep_clause(text: str) -> str: + """Append a keep-the-subject clause when the prompt lacks one.""" + body = str(text or "").strip() + if not body: + return body + if re.search(r"\b(keep|preserve|leave|unchanged|identical|same)\b", body, re.I): + return body + return body.rstrip(". ") + ". Keep the main subject and everything else in the image exactly the same." + + +def _mask_label(args: Dict[str, Any], prompt: str) -> str: + """Best-effort object noun for grounding a regional removal.""" + label = str(args.get("label") or args.get("object") or "").strip() + if label: + return label + m = re.search( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out)\s+(?:the\s+|a\s+|an\s+)?" + r"([a-z][a-z \-]{1,30}?)(?:\s+(?:on|in|at|from|to|near|behind|beside|by)\b|[.,;!?]|$)", + prompt or "", + re.I, + ) + if m: + noun = m.group(1).strip().lower() + if noun and noun not in {"background", "bg"}: + return noun + for noun in ("person", "people", "man", "woman", "car", "dog", "sign", "logo"): + if re.search(rf"\b{re.escape(noun)}\b", prompt or "", re.I): + return "person" if noun in {"person", "people", "man", "woman"} else noun + return "object" + + +class PlanEditTool: + """Plan an image edit as ordered editor operations (no side effects).""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + prompt = str(args.get("prompt") or "").strip() + if not prompt: + return "plan_edit: invalid", {"error": "plan_edit needs a non-empty `prompt`.", "exit_code": 1} + state = args.get("state") if isinstance(args.get("state"), dict) else {} + try: + data = await _post( + "/api/editor/plan", + {"prompt": prompt, "state": state, "mode": "plan", "session": _session(args, ctx)}, + ctx.get("owner"), + ) + except Exception as exc: + return "plan_edit: error", {"error": f"plan_edit failed: {exc}", "exit_code": 1} + steps = data.get("steps") or [] + reply = str(data.get("reply") or "") + return f"plan_edit: {len(steps)} step(s)", { + "output": reply or f"Planned {len(steps)} editor step(s).", + "reply": reply, + "planner": data.get("planner"), + "steps": steps, + "exit_code": 0, + } + + +class RunEditorCommandTool: + """Run a single validated editor op in the browser via the owner bridge.""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + op = str(args.get("op") or "").strip() + if not op: + return "run_editor_command: invalid", {"error": "run_editor_command needs `op`.", "exit_code": 1, "completed": False} + params = args.get("params") if isinstance(args.get("params"), dict) else {} + wait = bool(args.get("wait", True)) + payload = { + "session": _session(args, ctx), + "op": op, + "params": params, + "wait": wait, + "timeout": float(args.get("timeout") or 60), + } + try: + data = await _post("/api/editor/bridge/command", payload, ctx.get("owner")) + except Exception as exc: + return f"run_editor_command: {op}", {"error": f"run_editor_command failed: {exc}", "exit_code": 1, "completed": False, "status": "failed"} + + is_queued = bool(data.get("queued")) or data.get("status") == "queued" + if is_queued and (not wait or not data.get("ok")): + summary = json.dumps({"status": "queued", "queued": data.get("queued"), "session": data.get("session")}, ensure_ascii=False) + return f"run_editor_command: {op}", { + "output": summary, + "queued": data.get("queued"), + "status": "queued", + "pending": True, + "completed": False, + "ok": False, + "exit_code": 1, + } + + ok = bool(data.get("ok") is True and data.get("completed", True) is not False) + status = data.get("status") or ("completed" if ok else "failed") + result = data.get("result") + summary = json.dumps({"ok": ok, "status": status, "error": data.get("error"), "result": result}, ensure_ascii=False) + return f"run_editor_command: {op}", { + "output": summary[:4000], + "result": result, + "screenshot": data.get("screenshot"), + "status": status, + "completed": ok, + "ok": ok, + "exit_code": 0 if ok else 1, + } + + +class GetEditorScreenshotTool: + """Capture the current editor canvas as a data URL.""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + payload = { + "session": _session(args, ctx), + "op": "screenshot", + "params": {}, + "wait": True, + "timeout": float(args.get("timeout") or 30), + } + try: + data = await _post("/api/editor/bridge/command", payload, ctx.get("owner"), timeout=45) + except Exception as exc: + return "get_editor_screenshot: error", {"error": f"get_editor_screenshot failed: {exc}", "exit_code": 1} + result = data.get("result") if isinstance(data.get("result"), dict) else {} + shot = data.get("screenshot") or result.get("dataUrl") + if not shot: + return "get_editor_screenshot: unavailable", { + "error": "No editor screenshot was returned (is the editor open?).", + "exit_code": 1, + } + return "get_editor_screenshot", {"output": "Captured the editor canvas.", "screenshot": shot, "exit_code": 0} + + +class AiEditImageTool: + """Apply an AI image transform (remove-bg/upscale/denoise/enhance/inpaint).""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + _t = {"start": time.perf_counter()} + op = str(args.get("op") or "").strip().lower() + if op not in _AI_OPS: + return "ai_edit_image: invalid", { + "error": f"ai_edit_image `op` must be one of {sorted(_AI_OPS)}.", + "exit_code": 1, + } + owner = ctx.get("owner") + if op in ("inpaint", "generative") and not _provider_available(owner): + return "ai_edit_image: provider required", { + "error": _provider_message(), + "failure_kind": "image_provider_unavailable", + "needs_provider": True, + "exit_code": 1, + } + session = _session(args, ctx) + image = _as_data_url(args.get("image")) + if image is None and isinstance(args.get("image"), str) and args["image"].startswith("/"): + # A same-origin gallery/attachment URL: fetch it (owner-scoped) as bytes. + if not _image_reference_path(args["image"]): + return "ai_edit_image: invalid image", {"error": "Invalid image reference.", "exit_code": 1} + try: + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.get(f"{_INTERNAL_BASE}{args['image']}", headers=_internal_headers(owner)) + resp.raise_for_status() + if not resp.headers.get("content-type", "").lower().startswith("image/"): + return "ai_edit_image: invalid image", {"error": "Image reference is not an image.", "exit_code": 1} + image = _as_data_url("data:image/png;base64," + base64.b64encode(resp.content).decode()) + except Exception: + image = None + used_canvas = False + if not image: + shot = await GetEditorScreenshotTool().execute(json.dumps({"session": session}), ctx) + image = (shot[1] or {}).get("screenshot") if isinstance(shot, tuple) else None + used_canvas = bool(image) + if not image: + return "ai_edit_image: no image", { + "error": "ai_edit_image needs an `image` (data URL or same-origin URL) or an open editor.", + "exit_code": 1, + } + _t["image"] = time.perf_counter() + payload: Dict[str, Any] = {"image": image} + model_choice = str(args.get("model") or "").strip() + mask_used = False + place_mode: Optional[str] = None + if op == "upscale": + payload["scale"] = int(args.get("scale") or 2) + elif op == "denoise": + payload["strength"] = float(args.get("strength") or 0.5) + elif op in ("inpaint", "generative"): + payload["prompt"] = str(args.get("prompt") or "").strip() + if model_choice: + # inpaint_proxy reads `_model` to route to a specific model. + payload["_model"] = model_choice + mode = str(args.get("mode") or "instruction").strip().lower() + _steps = int(args.get("steps") or 0) + if _steps > 0: + payload["steps"] = _steps + mask_used = False + mask = None + if mode == "masked": + mask = _as_data_url(args.get("mask")) + if not mask: + mask = await _fetch_selection_mask(owner, session) + # An empty/tiny SAM mask on small figures is useless: drop it. + if mask and _mask_white_fraction(mask) < _MIN_MASK_FRACTION: + mask = None + if not mask: + # Fall back to the grounding detector (OWL-ViT): a dilated + # box mask around the matching object. + box = await _ground_box(owner, image, _mask_label(args, payload["prompt"])) + if box: + candidate = _rect_mask_data_url(image, box, pad_frac=0.15) + if candidate and _mask_white_fraction(candidate) >= _MIN_MASK_FRACTION: + mask = candidate + if mask: + payload["mask"] = mask + payload["mode"] = "masked" + mask_used = True + else: + # Nothing usable to mask -> whole-image instruction fallback + # rather than placing a transparent/empty layer. + mode = "instruction" + if mode != "masked": + # Instruction mode: whole-image edit with no tight mask. Send a + # full-white mask so the edit endpoint accepts the request and + # regenerates the whole frame under the instruction prompt, which + # names what must stay unchanged. This avoids the mask-compositing + # halos that loose selections caused. + payload["mode"] = "instruction" + payload.pop("mask", None) + payload["prompt"] = _ensure_keep_clause(payload["prompt"]) + try: + from PIL import Image as _PILImage + + _raw = _data_url_bytes(image) + _w, _h = _PILImage.open(io.BytesIO(_raw)).size + _mbuf = io.BytesIO() + _PILImage.new("L", (_w, _h), 255).save(_mbuf, "PNG") + payload["mask"] = "data:image/png;base64," + base64.b64encode(_mbuf.getvalue()).decode() + except Exception: + pass + place_mode = "masked" if mask_used else "instruction" + _t["prep"] = time.perf_counter() + try: + data = await _post(_AI_OPS[op], payload, owner) + except Exception as exc: + logger.info( + "image-op ai_edit_image op=%s stage=provider FAILED image_ms=%.1f prep_ms=%.1f total_ms=%.1f err=%s", + op, (_t["image"] - _t["start"]) * 1000.0, (_t["prep"] - _t["image"]) * 1000.0, + (time.perf_counter() - _t["start"]) * 1000.0, exc, + ) + return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) failed: {exc}", "exit_code": 1} + _t["provider"] = time.perf_counter() + result = data.get("image") or data.get("data") + data_url = _as_data_url(result) + if not data_url: + return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) returned no image.", "exit_code": 1} + if _image_is_blank(data_url): + return f"ai_edit_image: {op}", { + "error": f"ai_edit_image ({op}) produced an empty layer.", + "failure_kind": "empty_result", + "exit_code": 1, + } + + info: Dict[str, Any] = { + "output": f"Applied AI edit '{op}'" + (" to the current canvas." if used_canvas else "."), + "op": op, + "model": data.get("model") or model_choice or "", + "exit_code": 0, + } + if _incognito(ctx): + info["image_url"] = data_url + else: + try: + info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), f"ai_edit_image:{op}", f"ai_edit_image:{op}")) + except Exception as exc: + logger.warning("ai_edit_image persist failed: %s", exc) + _t["persist"] = time.perf_counter() + if bool(args.get("place", True)): + placed = await _place_layer( + data_url, + f"AI {op}", + owner, + session, + masked=mask_used, + blend=str(args.get("blend") or "normal"), + opacity=float(args.get("opacity") or 100), + mode=place_mode, + ) + info["placed"] = placed + if placed: + info["output"] += " Placed the result as a new layer." + _t["place"] = time.perf_counter() + logger.info( + "image-op ai_edit_image op=%s mode=%s canvas=%s image_ms=%.1f mask_ms=%.1f " + "model_ms=%.1f persist_ms=%.1f place_ms=%.1f total_ms=%.1f", + op, place_mode or "instruction", used_canvas, + (_t["image"] - _t["start"]) * 1000.0, + (_t["prep"] - _t["image"]) * 1000.0, + (_t["provider"] - _t["prep"]) * 1000.0, + (_t["persist"] - _t["provider"]) * 1000.0, + (_t["place"] - _t["persist"]) * 1000.0, + (_t["place"] - _t["start"]) * 1000.0, + ) + return f"ai_edit_image: {op}", info + + +class GenerateImageLayerTool: + """Generate an image with the configured image model and place it as a layer.""" + + async def execute(self, content: str, ctx: dict): + args = _parse(content) + prompt = str(args.get("prompt") or "").strip() + if not prompt: + return "generate_image: invalid", {"error": "generate_image needs a non-empty `prompt`.", "exit_code": 1} + owner = ctx.get("owner") + if not _provider_available(owner): + return "generate_image: provider required", { + "error": _provider_message(), + "failure_kind": "image_provider_unavailable", + "needs_provider": True, + "exit_code": 1, + } + session = _session(args, ctx) + payload: Dict[str, Any] = {"prompt": prompt} + for key in ("model", "size", "quality"): + if args.get(key): + payload[key] = args[key] + try: + from src.ai_interaction import do_generate_image + + data = await do_generate_image( + json.dumps(payload, ensure_ascii=False), + session_id=session, + owner=owner, + incognito=True, + ) + except Exception as exc: + return "generate_image: error", {"error": f"generate_image failed: {exc}", "exit_code": 1} + result = data.get("image_url") or data.get("image") or data.get("data") + data_url = _as_data_url(result) + if not data_url: + return "generate_image", {"error": "generate_image returned no image.", "exit_code": 1} + info: Dict[str, Any] = { + "output": f"Generated an image for: {prompt[:120]}", + "image_size": data.get("image_size"), + "model": data.get("image_model") or str(args.get("model") or ""), + "exit_code": 0, + } + if _incognito(ctx): + info["image_url"] = data_url + else: + try: + info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), prompt, "generate_image")) + except Exception as exc: + logger.warning("generate_image persist failed: %s", exc) + if bool(args.get("place", True)): + placed = await _place_layer( + data_url, + str(args.get("name") or "Generated image"), + owner, + session, + masked=bool(args.get("mask")), + blend=str(args.get("blend") or "normal"), + opacity=float(args.get("opacity") or 100), + ) + info["placed"] = placed + if placed: + info["output"] += " Placed it as a new layer." + return "generate_image", info diff --git a/src/agent_tools/web_tools_prewave3.py b/src/agent_tools/web_tools_prewave3.py new file mode 100644 index 000000000..700ab4a85 --- /dev/null +++ b/src/agent_tools/web_tools_prewave3.py @@ -0,0 +1,3863 @@ +import asyncio +import base64 +import contextlib +import contextvars +import inspect +import io +import json +import os +import re +import signal +import shutil +import sys +import tempfile +import time +import html +import hashlib +import pwd +import urllib.parse +import urllib.request +import uuid +from pathlib import Path +from typing import Dict, Any + +from core import platform_compat +from src import browser_lifecycle, process_lifecycle +from src.constants import MAX_OUTPUT_CHARS + +PDF_EXTRACT_MAX_BYTES = 80_000_000 +_ACTIVE_BROWSER_SESSIONS: set[str] = set() + + +def _service_home() -> Path: + """Return the account home even when a task overrides ``HOME``.""" + + try: + return Path(pwd.getpwuid(os.getuid()).pw_dir) + except (KeyError, OSError): + return Path.home() + + +def _host_npm_roots() -> list[Path]: + """Return user npm roots visible to a runtime using an isolated HOME.""" + + roots = [_service_home() / ".npm"] + roots.extend(Path("/home").glob("*/.npm")) + roots.append(Path("/root/.npm")) + return list(dict.fromkeys(roots)) + + +def _accessible_glob(roots: list[Path], pattern: str): + """Yield matches while ignoring roots unreadable by the service account.""" + + for root in dict.fromkeys(roots): + try: + yield from root.glob(pattern) + except (OSError, PermissionError): + continue + + +def _browser_executable_candidates() -> list[Path]: + """Return executable Chromium builds from standard host cache layouts.""" + + homes = [_service_home()] + homes.extend(path for path in _accessible_glob([Path("/home")], "*") if path.is_dir()) + homes.append(Path("/root")) + patterns = ( + ".cache/ms-playwright/chromium-*/chrome-linux64/chrome", + ".cache/ms-playwright/chromium_headless_shell-*/" + "chrome-headless-shell-linux64/chrome-headless-shell", + ".chromium-browser-snapshots/chromium/linux-*/chrome-linux/chrome", + ) + candidates = [ + path + for pattern in patterns + for path in _accessible_glob(list(dict.fromkeys(homes)), pattern) + if path.is_file() and os.access(path, os.X_OK) + ] + return sorted(dict.fromkeys(candidates), reverse=True) + + +def _bounded_browser_identity(value: str, *, max_length: int = 20) -> str: + """Return an agent-browser identity safe for Unix socket paths.""" + + safe = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(value or "")).strip("-._") + if safe == value and 0 < len(safe) <= max_length: + return safe + return hashlib.sha256(str(value).encode("utf-8")).hexdigest()[:max_length] + + +def _scoped_browser_session(namespace: str, session_id: str) -> str: + """Build one upstream-compatible session key for runtime isolation.""" + + scope = f"{str(namespace or 'odysseus-ui')}\0{str(session_id or '')}" + return f"ody-{_bounded_browser_identity(scope)}" + + +def _browser_namespace(env: dict[str, str] | None) -> str: + return str( + (env or {}).get("ODYSSEUS_BROWSER_NAMESPACE") + or os.getenv("ODYSSEUS_BROWSER_NAMESPACE", "odysseus-ui") + ).strip() or "odysseus-ui" + + +# Browser CLI processes started by the current private_browser call, so a +# cancellation can stop every client it spawned, not only the main command. +_BROWSER_CALL_PROCS: contextvars.ContextVar[list | None] = contextvars.ContextVar( + "_BROWSER_CALL_PROCS", default=None +) + + +async def _spawn_browser_cli(*command, **kwargs): + proc = await asyncio.create_subprocess_exec(*command, **kwargs) + # Identity taken while we hold the unreaped child: teardown later signals + # its group only while this identity still verifies, never on the pid + # alone (the event loop may reap it before returncode is observed). + pid = getattr(proc, "pid", None) + if isinstance(pid, int) and pid > 0: + with contextlib.suppress(Exception): + proc._ody_identity = process_lifecycle.ProcessIdentity.capture(pid, pgid=pid) + tracked = _BROWSER_CALL_PROCS.get() + if tracked is not None: + tracked.append(proc) + return proc + + +def _browser_pid_file_candidates( + runtime_dir: Path, namespace: str, session_id: str | None +) -> list[Path]: + """Return only the daemon pid files owned by one browser runtime. + + Current agent-browser stores ``--session`` state directly below its + runtime directory. Older builds used a namespace/run subdirectory, so + retain that layout as a compatibility fallback. When no session is + supplied, only the legacy namespace directory is eligible; never sweep + all upstream sessions from the shared root. + """ + + browser_root = runtime_dir / "agent-browser" + legacy_run = ( + browser_root / "namespaces" / _bounded_browser_identity(namespace) / "run" + ) + if not session_id: + return list(legacy_run.glob("ody-*.pid")) if legacy_run.is_dir() else [] + + scoped = _scoped_browser_session(namespace, session_id) + candidates = [browser_root / f"{scoped}.pid"] + if legacy_run.is_dir(): + candidates.extend( + [ + legacy_run / f"ody-{_bounded_browser_identity(session_id)}.pid", + legacy_run / f"{scoped}.pid", + ] + ) + return list(dict.fromkeys(candidates)) + + +# Linux exposes one command line per pid under /proc; macOS and Windows do not. +# Kept as a module attribute so the procfs-dependent paths stay testable on a +# host that has no procfs, and on one that does. + + +def _process_command_line(pid: int) -> str | None: + """Command line of a running process, or ``None`` when it cannot be read. + + ``None`` means "this host cannot tell", not "the process is gone". Off + Linux there is no procfs to read a command line from, so callers must not + treat it as proof that the process exited. + """ + + try: + return (platform_compat.PROC_ROOT / str(pid) / "cmdline").read_bytes().replace( + b"\0", b" " + ).decode("utf-8", errors="replace") + except (OSError, UnicodeError): + return None + + +def _process_is_alive(pid: int) -> bool: + """Whether a pid currently exists. + + Delegates to ``core.platform_compat.pid_alive`` rather than probing with + ``os.kill(pid, 0)`` directly. That probe is POSIX-only: CPython's Windows + ``os.kill`` calls ``TerminateProcess(handle, sig)`` for any signal other + than CTRL_C / CTRL_BREAK, so it would *kill* the daemon it is asked about. + Windows is also where there is no procfs, which is precisely when this + function gets called at all. + + ``pid_alive`` reads False for a pid that ``os.kill`` reports with + ``PermissionError`` — a live process owned by another user. Neither caller + here wants a different answer: the sweep only unlinks a pid file it wrote + itself, and treating somebody else's pid as "not our daemon" is the safe + reading in both. + """ + + return platform_compat.pid_alive(pid) + +_SCHOLARLY_METADATA_CUE_RE = re.compile( + r"\b(?:accept(?:ed|ance)?|publish(?:ed|ing|cation)?|venue|conference|" + r"journal|proceedings|doi)\b", + re.IGNORECASE, +) +_EXPLICIT_ARXIV_ID_RE = re.compile( + r"\barxiv(?:\.org)?\b.{0,24}\b\d{4}\.\d{4,5}(?:v\d+)?\b", + re.IGNORECASE, +) +_SCHOLARLY_SUBJECT_CUE_RE = re.compile( + r"\b(?:paper|preprint|arxiv|benchmark|language model|vision-language|" + r"ICLR|ICML|CVPR|NeurIPS|ACL|EMNLP|AAAI|IEEE)\b", + re.IGNORECASE, +) +_DISTINCTIVE_SCHOLARLY_NAME_RE = re.compile( + r"\b(?:[A-Za-z][A-Za-z-]*\d[A-Za-z0-9.-]*|" + r"[A-Z][A-Za-z0-9]*-[A-Z][A-Za-z0-9]*)\b" +) + + +def _is_scholarly_metadata_query(query: str) -> bool: + text = str(query or "") + cues = { + match.group(0).casefold() + for match in _SCHOLARLY_METADATA_CUE_RE.finditer(text) + } + # Source discovery for a named paper should return ranked URLs/snippets, + # not download every result page. Full-page fetching can spend the entire + # tool timeout on one blocked publisher before the model ever sees the + # arXiv/official result it needs for pdf_extract. + if ( + re.search(r"\b(?:paper|preprint|arxiv)\b", text, re.IGNORECASE) + and re.search( + r"\b(?:table|figure|benchmark|results?|pdf|source|url)\b", + text, + re.IGNORECASE, + ) + ): + return True + if ( + _DISTINCTIVE_SCHOLARLY_NAME_RE.search(text) + and re.search(r"\b(?:table|figure|benchmark|scores?|results?)\b", text, re.IGNORECASE) + ): + return True + if not cues: + return False + return bool( + _EXPLICIT_ARXIV_ID_RE.search(text) + or _SCHOLARLY_SUBJECT_CUE_RE.search(text) + or len(cues) >= 3 + ) + + +def _is_official_site_metadata_query(query: str) -> bool: + """Canonical-homepage lookup needs ranked URLs, not downloaded pages.""" + + value = re.sub(r"\s+", " ", str(query or "")).strip() + return bool(re.fullmatch( + r"(?:the\s+)?official\s+.+?\s+(?:website|web\s*site|site|homepage)" + r"|.+?\s+official\s+(?:website|web\s*site|site|homepage)", + value, + re.IGNORECASE, + )) + + +def _format_search_metadata(query: str, results: list[dict]) -> tuple[str, list[dict]]: + sources = [ + {"url": str(item.get("url") or ""), "title": str(item.get("title") or "")} + for item in results + if item.get("url") + ] + parts = [f"WEB SEARCH METADATA RESULTS\nQuery: {query}"] + for index, item in enumerate(results, 1): + parts.extend([ + "", + f"[{index}] {item.get('title') or 'Untitled result'}", + f"URL: {item.get('url') or ''}", + f"Snippet: {str(item.get('snippet') or '')[:1200]}", + ]) + if not results: + parts.append("\nNo search results found.") + return "\n".join(parts), sources + + +def _looks_like_youtube_video_id(value: str) -> bool: + return bool(re.fullmatch(r"[A-Za-z0-9_-]{10,16}", str(value or "").strip())) + + +def _arxiv_listing_api_hint(url: str, error: str) -> str: + """Return an explicit recovery path for a blocked arXiv date listing.""" + if "406" not in str(error or ""): + return "" + parsed = urllib.parse.urlsplit(str(url or "")) + if parsed.hostname not in {"arxiv.org", "www.arxiv.org"}: + return "" + match = re.fullmatch(r"/list/([A-Za-z0-9.-]+)", parsed.path.rstrip("/")) + date = urllib.parse.parse_qs(parsed.query).get("date", [""])[0] + if not match or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date): + return "" + day = date.replace("-", "") + query = f"cat:{match.group(1)} AND submittedDate:[{day}0000 TO {day}2359]" + api_url = "https://export.arxiv.org/api/query?" + urllib.parse.urlencode({ + "search_query": query, "start": "0", "max_results": "100", + }) + return ( + " arXiv rejected its static listing (HTTP 406). The public Atom API is " + f"available for this category/day; call web_fetch on {api_url} to read " + "the dated feed, then inspect individual paper sources for details." + ) + + +class WebSearchTool: + async def execute(self, content: str, ctx: dict) -> dict: + from src.search import comprehensive_web_search, searxng_search_results + progress_cb = ctx.get("progress_cb") if isinstance(ctx, dict) else None + raw = content.strip() + query = raw + time_filter = None + max_pages = 5 + if raw.startswith("{"): + try: + parsed = json.loads(raw) + if isinstance(parsed, dict) and "query" in parsed: + query = str(parsed.get("query", "")).strip() + tf = parsed.get("time_filter") or parsed.get("freshness") + if isinstance(tf, str) and tf.lower() in ("day", "week", "month", "year"): + time_filter = tf.lower() + mp = parsed.get("max_pages") + if isinstance(mp, int) and 1 <= mp <= 10: + max_pages = mp + except json.JSONDecodeError: + pass + if not query: + query = raw.split("\n")[0].strip() + if time_filter is None: + from src.search_intent import inferred_search_publication_window + time_filter = inferred_search_publication_window(query) + loop = asyncio.get_running_loop() + if progress_cb: + await progress_cb({ + "elapsed_s": 0, + "tail": f"Searching web for: {query[:160]}", + }) + try: + if _is_scholarly_metadata_query(query) or _is_official_site_metadata_query(query): + results = await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: searxng_search_results(query, max_pages, **({'time_filter': time_filter} if time_filter else {})), + ), + timeout=30, + ) + text, sources = _format_search_metadata(query, results) + else: + text, sources = await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: comprehensive_web_search( + query, + max_pages=max_pages, + time_filter=time_filter, + return_sources=True, + ), + ), + timeout=30, + ) + except asyncio.TimeoutError: + # Comprehensive search also downloads several result pages. A + # slow or hostile publisher must not erase the ranked search + # evidence that was already available. Fall back to the metadata + # path so the agent can choose a source and continue with + # web_fetch/private_browser. Keep this bounded independently: the + # abandoned executor thread may still be winding down. + try: + results = await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: searxng_search_results(query, max_pages, **({'time_filter': time_filter} if time_filter else {})), + ), + timeout=12, + ) + text, sources = _format_search_metadata(query, results) + if sources: + output = text[:MAX_OUTPUT_CHARS] if len(text) > MAX_OUTPUT_CHARS else text + output += "\n\n" + return { + "output": output, + "exit_code": 0, + "evidence_status": "available", + "degraded_mode": "metadata_after_content_timeout", + } + except Exception: + pass + return { + "error": f"web_search timed out after 30s: {query[:200]}", + "exit_code": 1, + } + except Exception as e: + return { + "error": f"web_search failed: {type(e).__name__}: {str(e) or 'no details'}", + "exit_code": 1, + "untrusted_content": True, + } + from .weather_tools import WeatherTool, weather_location_from_query + weather_location = weather_location_from_query(query) + if not sources and time_filter and weather_location: + # A forecast or current fact need not live on a newly published page. + try: + text, sources = await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: comprehensive_web_search( + query, max_pages=max_pages, time_filter=None, + return_sources=True, + ), + ), + timeout=20, + ) + except Exception: + pass + if not sources: + from src.turn_contract import active_turn_contract + contract = active_turn_contract() + policy = ctx.get("tool_policy") if isinstance(ctx, dict) else None + weather_allowed = ( + weather_location + and "get_weather" not in (ctx.get("disabled_tools") or ()) + and not (policy and policy.blocks("get_weather")) + and not (contract and not contract.permits("get_weather")) + ) + if weather_allowed: + weather = await WeatherTool().execute(json.dumps({"location": weather_location}), ctx) + if weather.get("exit_code") == 0: + return weather + if progress_cb: + await progress_cb({ + "elapsed_s": 30, + "tail": "Search completed; preparing sources.", + }) + # Compact the complete report before the transport cap. Otherwise + # repeated summaries from early sources permanently erase later pages. + from src.search_passages import bounded_search_observation + text = bounded_search_observation(text, MAX_OUTPUT_CHARS) + output = text[:MAX_OUTPUT_CHARS] if len(text) > MAX_OUTPUT_CHARS else text + if sources: + output += "\n\n" + return {"output": output, "exit_code": 0, + "evidence_status": "available" if sources else "empty"} + +class WebFetchTool: + _MAX_BATCH_URLS = 12 + _MAX_BATCH_CONCURRENCY = 4 + + @staticmethod + def _query_from_request(request_text: str) -> str: + """Extract distinctive document/table terms from the active request.""" + candidates = re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{2,}", request_text or "") + generic = { + "http", "https", "arxiv.org", "pdf", "pdfs", "www", "com", "org", + "please", "download", "read", "create", "save", "write", "workspace", + "table", "tables", "report", "benchmark", "benchmarks", "score", "scores", + "model", "models", "file", "chart", "data", "using", "from", "with", + } + selected: list[str] = [] + for token in candidates: + folded = token.casefold() + distinctive = ( + any(ch.isdigit() for ch in token) + or any(ch.isupper() for ch in token[1:]) + or "-" in token + or folded.endswith("qa") + ) + if distinctive and folded not in generic and token not in selected: + selected.append(token) + return ", ".join(selected[:20]) + + @staticmethod + def _focused_passages(text: str, query: str, max_chars: int) -> str: + """Select bounded page/line passages matching long-document terms.""" + segments = [term.strip() for term in re.split(r"[,|;\n]+", query) if term.strip()] + stopwords = { + "benchmark", "table", "evaluation", "scores", "score", "model", + "paper", "document", "accuracy", "results", "result", + } + terms: list[str] = [] + for segment in segments: + if len(segments) > 1 and len(segment) >= 2: + terms.append(segment) + for token in re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{2,}", segment): + if token.casefold() not in stopwords and token not in terms: + terms.append(token) + terms = terms[:16] + if not terms: + return text + lines = text.splitlines() + anchors: list[int] = [] + for index, line in enumerate(lines): + low = line.casefold() + if any(term.casefold() in low for term in terms): + anchors.append(index) + if not anchors: + return f"[No passages matched query: {query}]" + # Keep independent windows around each hit. Merging nearby hits can + # accidentally turn a dense PDF table/page into one giant passage and + # clip the target row at the output boundary. + windows: list[tuple[int, int]] = [] + for anchor in anchors: + table_start = None + if len(re.findall(r"\b\d+(?:\.\d+)?\b", lines[anchor])) >= 4: + for candidate in range(anchor, max(-1, anchor - 81), -1): + if re.match(r"\s*Table\s+\d+", lines[candidate], re.IGNORECASE): + table_start = candidate + break + interval = ( + (table_start, min(len(lines), anchor + 9)) + if table_start is not None + else (max(0, anchor - 28), min(len(lines), anchor + 15)) + ) + if interval not in windows: + windows.append(interval) + ranked_passages = [] + for start, end in windows: + passage = "\n".join(lines[start:end]) + low = passage.casefold() + coverage = sum(1 for term in terms if term.casefold() in low) + occurrences = sum(min(low.count(term.casefold()), 8) for term in terms) + numeric_density = min(len(re.findall(r"\b\d+(?:\.\d+)?\b", passage)), 40) + table_rows = 0 + for line in passage.splitlines(): + line_low = line.casefold() + if ( + any(term.casefold() in line_low for term in terms) + and len(re.findall(r"\b\d+(?:\.\d+)?\b", line)) >= 4 + ): + table_rows += 1 + score = coverage * 100 + occurrences * 4 + numeric_density + min(table_rows, 4) * 200 + # PDF table extraction can collapse a full page into one enormous + # line. Bound each candidate so an earlier broad table cannot + # consume the entire result before the exact matching table. + passage_cap = max(2000, min(8000, max_chars // 3)) + if len(passage) > passage_cap: + # HTML-to-text can collapse an entire arXiv page into one line. + # Choose the densest term/numeric window instead of blindly + # retaining the abstract at the head of that line. + candidates: list[tuple[int, int, str]] = [] + passage_low = passage.casefold() + for term in terms: + term_low = term.casefold() + cursor = 0 + for _ in range(8): + position = passage_low.find(term_low, cursor) + if position < 0: + break + window_start = max(0, min( + len(passage) - passage_cap, + position - passage_cap // 2, + )) + window = passage[window_start:window_start + passage_cap] + window_low = window.casefold() + window_coverage = sum( + 1 for candidate in terms + if candidate.casefold() in window_low + ) + window_numbers = min( + len(re.findall(r"\b\d+(?:\.\d+)?\b", window)), 300 + ) + candidates.append(( + window_coverage * 1000 + + window_numbers + + len(term) * 10 + - window_start // 100, + -window_start, + window, + )) + cursor = position + max(1, len(term_low)) + if candidates: + passage = max(candidates, key=lambda item: (item[0], item[1]))[2] + else: + passage = passage[:passage_cap] + passage += "\n[...passage windowed around query terms]" + ranked_passages.append((score, start, passage)) + ranked_passages.sort(key=lambda item: (-item[0], item[1])) + passages: list[str] = [] + for _score, _index, passage in ranked_passages: + # Collapse duplicate/near-identical windows without merging them. + normalized = re.sub(r"\s+", " ", passage).strip() + if any(normalized in re.sub(r"\s+", " ", prior) for prior in passages): + continue + passages.append(passage) + if len(passages) >= 1: + break + selected = "\n\n--- matching passage ---\n\n".join(passages) + if len(selected) > max_chars: + selected = selected[:max_chars] + "\n\n[...focused passages truncated]" + return f"[Focused passages for: {query}]\n\n{selected}" + + async def _execute_batch(self, payload: dict, ctx: dict) -> dict: + """Fetch several known URLs concurrently with bounded combined output.""" + raw_urls = payload.get("urls") + if not isinstance(raw_urls, list) or not raw_urls: + return { + "error": "web_fetch: urls must be a non-empty array of at most 12 URLs", + "exit_code": 1, + } + if len(raw_urls) > self._MAX_BATCH_URLS: + return { + "error": "web_fetch: urls must contain at most 12 URLs", + "exit_code": 1, + } + + shared_query = str(payload.get("query") or "").strip() + shared_full = payload.get("full") is True + requests: list[dict[str, Any]] = [] + for index, item in enumerate(raw_urls): + if isinstance(item, str): + request = {"url": item, "query": shared_query, "full": shared_full} + elif isinstance(item, dict): + request = { + "url": str(item.get("url") or "").strip(), + "query": str(item.get("query") or shared_query).strip(), + "full": item.get("full") is True or shared_full, + } + else: + return { + "error": f"web_fetch: urls item {index + 1} must be a URL string or object", + "exit_code": 1, + } + if not request["url"]: + return { + "error": f"web_fetch: urls item {index + 1} is missing url", + "exit_code": 1, + } + requests.append(request) + + semaphore = asyncio.Semaphore(self._MAX_BATCH_CONCURRENCY) + child_ctx = dict(ctx) if isinstance(ctx, dict) else {} + child_ctx.pop("progress_cb", None) + + async def fetch_one(request: dict[str, Any]) -> dict: + async with semaphore: + return await self.execute(json.dumps(request), child_ctx) + + results = await asyncio.gather(*(fetch_one(request) for request in requests)) + per_item_cap = max(800, min(6000, (MAX_OUTPUT_CHARS - 1000) // len(results))) + sections: list[str] = [] + successful = 0 + summaries: list[dict[str, Any]] = [] + for index, (request, result) in enumerate(zip(requests, results), start=1): + ok = result.get("exit_code") == 0 + if ok: + successful += 1 + body = str(result.get("output") or "") + else: + body = "ERROR: " + str(result.get("error") or "fetch failed") + if len(body) > per_item_cap: + body = body[:per_item_cap] + "\n[...batch item truncated]" + sections.append(f"## URL {index}: {request['url']}\n{body}") + summaries.append({ + "url": request["url"], + "exit_code": int(result.get("exit_code", 1)), + }) + output = "\n\n".join(sections) + if len(output) > MAX_OUTPUT_CHARS: + output = output[:MAX_OUTPUT_CHARS] + "\n\n[...batch output truncated]" + return { + "output": output, + "batch_results": summaries, + "successful": successful, + "requested": len(requests), + "exit_code": 0 if successful else 1, + } + + async def execute(self, content: str, ctx: dict) -> dict: + from src.search.content import fetch_webpage_content + from src.constants import WEB_FETCH_HARD_MAX_BYTES + raw = content.strip() + if raw.startswith("{"): + try: + batch_payload = json.loads(raw) + except json.JSONDecodeError: + batch_payload = None + if isinstance(batch_payload, dict) and "urls" in batch_payload: + return await self._execute_batch(batch_payload, ctx) + url = "" + max_bytes = None + query = "" + if raw.startswith("{"): + try: + parsed = json.loads(raw) + if isinstance(parsed, dict): + url = str(parsed.get("url") or "").strip() + query = str(parsed.get("query") or "").strip() + # Download-budget override (#3812): "full": true raises the + # budget to the hard cap; an explicit max_bytes is clamped + # to the hard cap downstream. Default stays the soft cap. + if parsed.get("full") is True: + max_bytes = WEB_FETCH_HARD_MAX_BYTES + mb = parsed.get("max_bytes") + if isinstance(mb, int) and mb > 0: + max_bytes = mb + except json.JSONDecodeError: + url = "" + if not url: + url = raw.split("\n")[0].strip() + local_path = url + if local_path.lower().startswith("file:///workspace/"): + local_path = local_path[7:] + if local_path.startswith("/workspace/"): + suffix = Path(local_path.split("?", 1)[0].split("#", 1)[0]).suffix.lower() + if suffix in {".html", ".htm"}: + return { + "error": ( + "web_fetch: local HTML requires private_browser so rendered DOM and " + "page errors are inspected" + ), + "exit_code": 1, + } + if suffix in { + ".avi", ".bmp", ".gif", ".jpeg", ".jpg", ".m4v", ".mkv", + ".mov", ".mp4", ".mpeg", ".mpg", ".png", ".webm", ".webp", + }: + return { + "error": "web_fetch: local visual media requires inspect_media", + "exit_code": 1, + } + from src.agent_tools.filesystem_tools import ReadFileTool + + result = await ReadFileTool().execute(local_path, ctx) + if result.get("exit_code") == 0: + result = dict(result) + result["output"] = ( + "Argument note: Read the local workspace path with read_file; " + "use web_fetch only for HTTP(S) URLs.\n" + str(result.get("output") or "") + ) + return result + if not url or url.startswith("{") or any(c in url for c in (" ", "\t", "\n")): + return {"error": "web_fetch: provide a single URL or domain, e.g. example.com", "exit_code": 1} + low = url.lower() + if "://" in low and not low.startswith(("http://", "https://")): + return {"error": f"web_fetch: unsupported URL scheme (only http/https): {url[:80]}", "exit_code": 1} + if not low.startswith(("http://", "https://")): + url = "https://" + url + if re.search(r"(?:\.pdf(?:[?#]|$)|arxiv\.org/pdf/)", url, re.IGNORECASE) and not query: + runtime_context = ctx.get("client_runtime_context") if isinstance(ctx, dict) else None + request_text = ( + str(runtime_context.get("request_text") or "") + if isinstance(runtime_context, dict) else "" + ) + query = self._query_from_request(request_text) + if not query: + return { + "error": ( + "web_fetch: this is a PDF. Use pdf_extract with this URL and a query " + "that names the target model, metrics, or table; do not download it " + "with Python/curl and do not estimate missing values." + ), + "exit_code": 1, + } + loop = asyncio.get_running_loop() + try: + def _fetch(): + kwargs = {"timeout": 10} + try: + sig = inspect.signature(fetch_webpage_content) + if "max_bytes" in sig.parameters: + kwargs["max_bytes"] = max_bytes + except (TypeError, ValueError): + # Some deployed/test shims may not expose a signature. + # Prefer compatibility over failing the whole fetch. + pass + return fetch_webpage_content(url, **kwargs) + + result = await asyncio.wait_for( + loop.run_in_executor(None, _fetch), + timeout=30, + ) + except asyncio.TimeoutError: + return {"error": f"web_fetch: timed out fetching {url}", "exit_code": 1} + except Exception as e: + return {"error": f"web_fetch: {url}: {e}", "exit_code": 1} + err = result.get("error") + text = (result.get("linked_content") or result.get("content") or "").strip() + title = result.get("title") or "" + + if not text: + if err: + arxiv_hint = _arxiv_listing_api_hint(url, str(err)) + return { + "error": f"web_fetch: {url}: {err}{arxiv_hint}", + "exit_code": 1, + "untrusted_content": True, + } + return {"error": f"web_fetch: {url}: no readable text content (not HTML, or the page needs JS/login)", "exit_code": 1} + + # Tell the model when the download budget cut the body short and how + # to get the rest, instead of silently presenting a partial page as + # the whole thing. + size_note = "" + if result.get("truncated"): + fetched = result.get("fetched_bytes") or 0 + total = result.get("total_bytes") + total_txt = f" of {total:,} bytes" if total else "" + size_note = ( + f"[partial content: download stopped at {fetched:,} bytes{total_txt}. " + f'Re-call with {{"url": "{url}", "full": true}} to fetch up to ' + f"{WEB_FETCH_HARD_MAX_BYTES:,} bytes.]\n\n" + ) + + # The notice must lead the output so the MAX_OUTPUT_CHARS trim below can + # never drop it. The title is untrusted, uncapped page content, so a + # giant title ahead of the notice could push it out of range; keep the + # notice first and cap the title as a second guard. + if len(title) > 300: + title = title[:300] + "..." + header = (f"# {title}\n" if title else "") + f"Source: {url}\n\n" + if query: + text = self._focused_passages(text, query, MAX_OUTPUT_CHARS - len(header) - 500) + output = size_note + header + text + if len(output) > MAX_OUTPUT_CHARS: + output = output[:MAX_OUTPUT_CHARS] + ( + "\n\n[...truncated; re-call web_fetch with query terms to retrieve matching passages]" + if not query else "\n\n[...truncated]" + ) + return {"output": output, "exit_code": 0, "page_entries": result.get("page_entries") or []} + + +class PdfExtractTool: + """Extract focused, source-attributed passages from a PDF URL or workspace file. + + This deliberately builds on Odysseus' native web reader instead of adding + a benchmark transport. The separate semantic affordance keeps models from + downloading PDFs with ad-hoc Python and then guessing when extraction fails. + Local task PDFs use the same positioned reader, but are resolved through the + active workspace path policy rather than through an unrestricted filesystem + path. + """ + + @staticmethod + def _local_pdf_path(source: str) -> Path | None: + """Resolve a task-local PDF through the active workspace policy.""" + raw = str(source or "").strip() + if raw.lower().startswith("file://"): + parsed = urllib.parse.urlsplit(raw) + if parsed.netloc not in {"", "localhost"}: + return None + raw = urllib.parse.unquote(parsed.path) + if not raw.startswith("/workspace/") and not Path(raw).is_absolute(): + return None + try: + from src.tool_execution import _resolve_tool_path + + path = Path(_resolve_tool_path(raw)) + except (OSError, ValueError): + return None + return path if path.suffix.casefold() == ".pdf" else None + + @staticmethod + def _arxiv_html_url(url: str) -> str: + """Return the official HTML companion for an arXiv document URL.""" + match = re.search( + r"https?://(?:www\.)?arxiv\.org/(?:pdf|abs|html)/(?P\d{4}\.\d{4,5}(?:v\d+)?)", + str(url or ""), + re.IGNORECASE, + ) + return f"https://arxiv.org/html/{match.group('id')}" if match else "" + + @staticmethod + def _pdf_row_contains_term(row_text: str, term: str) -> bool: + """Match model names even when PDF glyph extraction splits punctuation.""" + row_folded = str(row_text or "").casefold() + term_folded = str(term or "").casefold() + if term_folded in row_folded: + return True + normalize = lambda value: re.sub(r"[^a-z0-9]+", "", value.casefold()) + normalized_term = normalize(term_folded) + return bool(normalized_term) and normalized_term in normalize(row_folded) + + @staticmethod + def _pdf_row_exact_term_match(row_text: str, term: str) -> bool: + """Match a complete model label without accepting a named variant. + + PDF text extraction may split punctuation (``LLaVA- Onevision``), so a + literal equality check is too strict. The broad table matcher above is + intentionally useful for locating candidate pages, but it also makes + ``DeepSeek-VL2`` match ``DeepSeek-VL2-Tiny``. Row resolution needs the + narrower relation: tolerate extracted punctuation while rejecting an + attached hyphen/slash variant suffix. + """ + row = str(row_text or "") + chunks = re.findall(r"[A-Za-z0-9]+", str(term or "")) + if not chunks: + return False + pattern = re.compile( + r"(? list[str]: + """Return compact digit-bearing aliases used by shared-family headers. + + Comparison tables often put the vendor/family in the caption and use + headers such as ``R1-Zero`` or ``o1``. Restrict aliases to suffixes + containing a digit so ordinary trailing words cannot become matches. + """ + parts = [part for part in re.split(r"-+", str(term or "")) if part] + aliases: list[str] = [] + for index in range(1, len(parts)): + alias = "-".join(parts[index:]) + if any(character.isdigit() for character in alias) and len(alias) >= 2: + aliases.append(alias) + return aliases + + @staticmethod + def _pdf_row_contains_model_alias(row_text: str, term: str) -> bool: + """Match an exact full model label or a bounded table-header alias.""" + if PdfExtractTool._pdf_row_exact_term_match(row_text, term): + return True + return any( + PdfExtractTool._pdf_row_exact_term_match(row_text, alias) + for alias in PdfExtractTool._pdf_model_suffix_aliases(term) + ) + + @staticmethod + def _bibliography_evidence( + page_columns: list[tuple[int, str, str]], query: str + ) -> str: + """Return ordered reference entries for bibliography-focused queries. + + Academic PDFs commonly use two columns. Reading the whole page in + layout order interleaves those columns, while generic query scoring can + rank citation-heavy body pages above the actual reference section. + ``page_columns`` keeps each column independent so references can be + reconstructed in normal reading order without knowing task page + numbers or paper titles. + """ + bibliography_intent = re.search( + r"\b(?:references?|bibliograph(?:y|ies)|bibtex|citation\s+list)\b", + str(query or ""), + re.IGNORECASE, + ) + if not bibliography_intent: + return "" + + heading_pattern = re.compile( + r"(?im)^\s*(?:references|bibliography)(?:\s+\d+)?\s*$" + ) + start_index = -1 + start_offset = 0 + for index, (_page_number, left, _right) in enumerate(page_columns): + heading = heading_pattern.search(left) + if heading: + start_index = index + start_offset = heading.start() + break + if start_index < 0: + return "" + + ordered_pages: list[tuple[int, str]] = [] + for index, (page_number, left, right) in enumerate( + page_columns[start_index:], start=start_index + ): + if index == start_index: + left = left[start_offset:] + text = "\n".join(part.strip() for part in (left, right) if part.strip()) + if text: + ordered_pages.append((page_number, text)) + if not ordered_pages: + return "" + + joined = "\n".join(text for _page_number, text in ordered_pages) + starts = list(re.finditer(r"(?m)^\s*\[(\d+)\]\s*", joined)) + numbered_entries: list[tuple[int, str]] = [] + for position, match in enumerate(starts): + end = starts[position + 1].start() if position + 1 < len(starts) else len(joined) + numbered_entries.append(( + int(match.group(1)), + joined[match.start():end].strip(), + )) + requested_range = re.search( + r"\b(?:references?|refs?|citations?)\s*(?:numbers?\s*)?" + r"\[?(\d{1,4})\]?\s*(?:through|to|[-–—])\s*\[?(\d{1,4})\]?", + str(query or ""), + re.IGNORECASE, + ) + if requested_range and numbered_entries: + first, last = sorted(( + int(requested_range.group(1)), + int(requested_range.group(2)), + )) + selected_entries = [ + entry for number, entry in numbered_entries + if first <= number <= last + ] + if selected_entries: + pages = f"{ordered_pages[0][0]}-{ordered_pages[-1][0]}" + body = ( + f"[Local PDF bibliography evidence, pages {pages}; " + f"reference entries {first}-{last}]\n" + + "\n\n".join(selected_entries) + ) + if len(body) > MAX_OUTPUT_CHARS - 500: + body = body[:MAX_OUTPUT_CHARS - 500] + "\n[...bibliography entries truncated]" + return body + + wants_arxiv = bool( + re.search(r"\b(?:arxiv|preprints?)\b", str(query or ""), re.IGNORECASE) + ) + if wants_arxiv: + # Keep complete numbered entries instead of returning every page. + # This is both more useful and prevents the bounded output cap from + # dropping late matching references. + entries: list[str] = [] + for _number, entry in numbered_entries: + if re.search(r"arxiv", entry, re.IGNORECASE): + entries.append(entry) + if entries: + pages = f"{ordered_pages[0][0]}-{ordered_pages[-1][0]}" + body = ( + f"[Local PDF bibliography evidence, pages {pages}; " + f"{len(entries)} arXiv/preprint entries]\n" + + "\n\n".join(entries) + ) + if len(body) > MAX_OUTPUT_CHARS - 500: + body = body[:MAX_OUTPUT_CHARS - 500] + "\n[...bibliography entries truncated]" + return body + + body = "\n\n".join( + f"[Local PDF bibliography evidence, page {page_number}]\n{text}" + for page_number, text in ordered_pages + ) + if len(body) > MAX_OUTPUT_CHARS - 500: + body = body[:MAX_OUTPUT_CHARS - 500] + "\n[...bibliography text truncated]" + return body + + @staticmethod + def _select_positioned_target_model( + model_terms: list[str], + rows: list[tuple[float, list[dict]]], + ) -> str: + """Choose the requested model that is actually present in selected PDF rows. + + Broad benchmark prompts often ask for several models in every PDF + request. Picking the longest/first requested model contaminates a GLM + or Seed table with the Qwen column. Score each requested model against + the selected positioned rows and prefer explicit in-table matches. + """ + if not model_terms: + return "" + table_text = " ".join( + str(word["text"]) + for _top, words in rows + for word in words + ) + table_folded = table_text.casefold() + table_normalized = re.sub(r"[^a-z0-9]+", "", table_folded) + scored: list[tuple[int, int, str]] = [] + for term in model_terms: + folded = str(term or "").casefold() + base = re.sub(r"-(?:\d+(?:\.\d+)?[BMK])$", "", term, flags=re.I) + score = 0 + for _top, words in rows: + row_text = " ".join(str(word["text"]) for word in words) + if PdfExtractTool._pdf_row_contains_term(row_text, term): + score += 100 + elif base and PdfExtractTool._pdf_row_contains_term(row_text, base): + score += 70 + if PdfExtractTool._pdf_row_contains_model_alias(row_text, term): + score += 60 + if "a22b" in folded: + row_folded = row_text.casefold() + normalized = re.sub(r"[^a-z0-9]+", "", row_folded) + if "a22b" in row_folded and ( + "qwen3" in row_folded + or "235b" in row_folded + or "qwen3vl" in normalized + ): + score += 80 + if "seed" in folded and "1.5" in folded: + if ( + "seed" in row_text.casefold() + and ("1.5-vl" in row_text.casefold() or "15vl" in re.sub(r"[^a-z0-9]+", "", row_text.casefold())) + ): + score += 80 + if "glm" in folded and "4.6" in folded: + if "glm" in row_text.casefold() and "4.6" in row_text.casefold(): + score += 80 + if "seed" in folded and "1.5" in folded: + if "seed" in table_folded and ( + "1.5-vl" in table_folded or "15vl" in table_normalized + ): + score += 90 + if "thinking" in folded and "thinking" in table_folded: + score += 25 + if "glm" in folded and "4.6" in folded: + if "glm" in table_folded and "4.6" in table_folded: + score += 90 + scored.append((score, len(term), term)) + best = max(scored) + return best[2] if best[0] > 0 else max(model_terms, key=len, default="") + + @staticmethod + def _looks_like_pdf_metric_term(token: str) -> bool: + """Identify benchmark metric labels, including compact table acronyms.""" + value = str(token or "") + folded = value.casefold() + if ( + folded.endswith("qa") + or "refcoco" in folded + or folded in { + "chartqa", + "docvqa", + "textvqa", + "ocrbench", + "countbench", + "blink", + "mmbench", + "mmstar", + "mmmu", + "mathvista", + } + ): + return True + # Papers frequently label table columns with short uppercase task + # acronyms (TR, AR, AO, AC, ...). Keep transport/document/model + # acronyms out so ordinary query prose does not become a fake header. + return ( + value.isupper() + and 2 <= len(value) <= 8 + and value not in {"PDF", "URL", "HTML", "HTTP", "HTTPS", "GPT", "LLM", "VLM"} + ) + + @staticmethod + def _pdf_query_tokens(query: str) -> list[str]: + """Tokenize table queries without dropping two-letter metric labels.""" + return re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{1,}", str(query or "")) + + @staticmethod + def _looks_like_pdf_model_term(token: str) -> bool: + """Separate compact model identifiers from ordinary hyphenated prose.""" + value = str(token or "") + if ( + any(character.isdigit() for character in value) + and any(character.isalpha() for character in value) + ): + return True + return "-" in value and sum(character.isupper() for character in value) >= 2 + + @staticmethod + def _pdf_words_contain_metric(words: list[dict], metric: str) -> bool: + """Match a metric as a table cell, not inside ordinary prose words.""" + normalize = lambda value: re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) + target = normalize(metric) + return bool(target) and any( + normalize(word.get("text")) == target for word in words + ) + + @staticmethod + def _pdf_rows_contain_table_label( + rows: list[tuple[float, list[dict]]], table_number: int + ) -> bool: + """Return whether positioned rows contain an exact numbered caption.""" + label = re.compile( + rf"^\s*table\s+{int(table_number)}(?:\D|$)", + re.IGNORECASE, + ) + return any( + label.search(" ".join(str(word.get("text") or "") for word in words)) + for _top, words in rows + ) + + @staticmethod + def _select_positioned_table_region( + rows: list[tuple[float, list[dict]]], + metric_terms: list[str], + model_terms: list[str], + requested_table_number: int | None, + ) -> list[tuple[float, list[dict]]]: + """Keep one physical table when a page contains adjacent tables. + + Joining coordinates across two tables on the same page can map a + header from one table to values in another. Real ``Table N`` caption + rows provide a generic boundary. Prefer an explicitly requested + table, otherwise rank regions by exact requested-row and metric + coverage; named variants receive only weak fallback credit. + """ + caption_pattern = re.compile(r"^\s*table\s+(\d{1,3})(?:\D|$)", re.I) + starts: list[tuple[int, int]] = [] + for index, (_top, words) in enumerate(rows): + row_text = " ".join(str(word.get("text") or "") for word in words) + match = caption_pattern.search(row_text) + if match: + starts.append((index, int(match.group(1)))) + if len(starts) < 2: + return rows + + regions: list[tuple[int, list[tuple[float, list[dict]]]]] = [] + for position, (start, number) in enumerate(starts): + end = starts[position + 1][0] if position + 1 < len(starts) else len(rows) + regions.append((number, rows[start:end])) + if requested_table_number is not None: + requested = [region for number, region in regions if number == requested_table_number] + if requested: + return requested[0] + + def _score(region: list[tuple[float, list[dict]]]) -> tuple[int, int]: + row_texts = [ + " ".join(str(word.get("text") or "") for word in words) + for _top, words in region + ] + exact_models = sum( + any(PdfExtractTool._pdf_row_exact_term_match(text, term) for text in row_texts) + for term in set(model_terms) + ) + broad_models = sum( + any(PdfExtractTool._pdf_row_contains_term(text, term) for text in row_texts) + for term in set(model_terms) + ) + metrics = sum( + any(PdfExtractTool._pdf_words_contain_metric(words, term) for _top, words in region) + for term in set(metric_terms) + ) + numeric = sum( + len(re.findall(r"\b\d+(?:\.\d+)?\b", text)) for text in row_texts + ) + return ( + exact_models * 1000 + broad_models * 100 + metrics * 300 + min(numeric, 99), + -len(region), + ) + + return max((region for _number, region in regions), key=_score) + + @staticmethod + def _positioned_table_evidence(url: str, query: str) -> str: + """Return compact PDF rows with x coordinates for column-safe reading.""" + try: + import pdfplumber + import httpx + from services.search.content import _PinnedTransport, _resolve_public_ips + from src.constants import WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT + except ImportError: + return "" + local_path = PdfExtractTool._local_pdf_path(url) + if local_path is not None: + if not local_path.is_file(): + return "" + pdf_bytes = local_path.read_bytes() + declared_size = len(pdf_bytes) + else: + ips = _resolve_public_ips(url) + headers = { + "User-Agent": WEB_FETCH_USER_AGENT, + "Accept": "application/pdf,*/*", + } + with httpx.Client( + headers=headers, + timeout=30, + follow_redirects=True, + transport=_PinnedTransport(ips[0]), + ) as client: + response = client.get(url) + response.raise_for_status() + pdf_bytes = response.content + declared = response.headers.get("content-length") + declared_size = int(declared) if declared and declared.isdigit() else 0 + if declared_size and declared_size > PDF_EXTRACT_MAX_BYTES: + return ( + f"[PDF too large for pdf_extract positioned table reader: " + f"{declared_size:,} bytes > {PDF_EXTRACT_MAX_BYTES:,} bytes]" + ) + if len(pdf_bytes) > PDF_EXTRACT_MAX_BYTES: + return ( + f"[PDF too large for pdf_extract positioned table reader: " + f"{len(pdf_bytes):,} bytes > {PDF_EXTRACT_MAX_BYTES:,} bytes]" + ) + tokens = PdfExtractTool._pdf_query_tokens(query) + requested_table_match = re.search( + r"\btable\s+(\d{1,3})\b", str(query or ""), re.IGNORECASE + ) + requested_table_number = ( + int(requested_table_match.group(1)) if requested_table_match else None + ) + metric_terms = [ + token for token in tokens + if PdfExtractTool._looks_like_pdf_metric_term(token) + ] + model_terms = [ + token for token in tokens + if token not in metric_terms + and PdfExtractTool._looks_like_pdf_model_term(token) + ] + normalize = lambda value: re.sub(r"[^a-z0-9]+", "", value.casefold()) + + def _row_contains_model_alias(row_text: str, term: str) -> bool: + row_folded = str(row_text or "").casefold() + term_folded = str(term or "").casefold() + if PdfExtractTool._pdf_row_contains_model_alias(row_text, term): + return True + if "a22b" in term_folded: + return "a22b" in row_folded and ( + "qwen3" in row_folded + or "235b" in row_folded + or "qwen3vl" in normalize(row_folded) + ) + return False + + candidates: list[tuple[int, int, list[tuple[float, list[dict]]]]] = [] + with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf: + for page_number, page in enumerate(pdf.pages, 1): + grouped: dict[float, list[dict]] = {} + for word in page.extract_words(x_tolerance=2, y_tolerance=2): + grouped.setdefault(round(float(word["top"]) / 3) * 3, []).append(word) + rows = [(top, sorted(words, key=lambda item: float(item["x0"]))) for top, words in grouped.items()] + anchors: list[float] = [] + score = 0 + for top, words in rows: + row_text = " ".join(str(word["text"]) for word in words) + low = row_text.casefold() + metric_hits = sum( + PdfExtractTool._pdf_words_contain_metric(words, term) + for term in metric_terms + ) + model_hits = sum( + PdfExtractTool._pdf_row_exact_term_match(row_text, term) + for term in model_terms + ) + alias_model_hits = sum( + _row_contains_model_alias(row_text, term) + for term in model_terms + ) + numeric_count = len(re.findall(r"\b\d+(?:\.\d+)?\b", row_text)) + starts_with_metric = bool(words) and any( + re.sub(r"[^a-z0-9]+", "", str(words[0]["text"]).casefold()) + == re.sub(r"[^a-z0-9]+", "", term.casefold()) + for term in metric_terms + ) + table_header_hit = ( + "capability" in low + and "benchmark" in low + ) + model_table_header_hit = bool( + alias_model_hits + and ( + low.strip().startswith("table ") + or low.strip().startswith("benchmark") + ) + ) + if metric_hits >= 2 or (metric_hits and numeric_count >= 3) or (model_hits and numeric_count >= 3): + anchors.append(top) + score += ( + metric_hits * 10 + + model_hits * 5 + + alias_model_hits * 20 + + min(numeric_count, 10) + + (50 if starts_with_metric else 0) + ) + elif model_table_header_hit: + score += alias_model_hits * 40 + if table_header_hit: + score += 15 + if anchors: + # Preserve the continuous table body between matching + # anchors. Selecting disjoint +/-100 bands dropped middle + # model rows in tall benchmark tables when the queried + # model name was split across PDF words and therefore did + # not become its own anchor. + table_top = max(0.0, min(anchors) - 100) + header_tops = [ + top for top, words in rows + if top <= min(anchors) + and ( + ( + "capability" in " ".join(str(word["text"]) for word in words).casefold() + and "benchmark" in " ".join(str(word["text"]) for word in words).casefold() + ) + or ( + "benchmark" in " ".join(str(word["text"]) for word in words).casefold() + and any( + _row_contains_model_alias( + " ".join(str(word["text"]) for word in words), + term, + ) + for term in model_terms + ) + ) + or ( + " ".join(str(word["text"]) for word in words).casefold().strip().startswith("table ") + and any( + _row_contains_model_alias( + " ".join(str(word["text"]) for word in words), + term, + ) + for term in model_terms + ) + ) + ) + ] + if header_tops: + table_top = min(table_top, max(header_tops) - 20) + table_bottom = max(anchors) + 100 + selected = [ + row for row in rows + if table_top <= row[0] <= table_bottom + ] + candidates.append((score, page_number, selected)) + if not candidates: + return "" + labeled_candidates = ( + [ + candidate for candidate in candidates + if PdfExtractTool._pdf_rows_contain_table_label( + candidate[2], requested_table_number + ) + ] + if requested_table_number is not None + else [] + ) + if labeled_candidates: + # A paper can repeat a table number in an appendix. The first + # exact caption is the canonical table for an unqualified request. + _score, page_number, rows = min( + labeled_candidates, key=lambda item: item[1] + ) + else: + _score, page_number, rows = max(candidates, key=lambda item: item[0]) + rows = PdfExtractTool._select_positioned_table_region( + rows, + metric_terms, + model_terms, + requested_table_number, + ) + resolved: dict[str, str] = {} + metric_positions: dict[str, float] = {} + header_candidates = [] + for _top, words in rows: + positions = { + metric: float(word["x0"]) + for word in words for metric in metric_terms + if re.sub(r"[^a-z0-9]+", "", str(word["text"]).casefold()) + == re.sub(r"[^a-z0-9]+", "", metric.casefold()) + } + if positions: + header_candidates.append((len(positions), positions)) + if header_candidates: + metric_positions = max(header_candidates, key=lambda item: item[0])[1] + + def _numbers(words: list[dict]) -> list[tuple[float, str]]: + return [ + (float(word["x0"]), str(word["text"])) + for word in words + if re.fullmatch(r"\d+(?:\.\d+)?", str(word["text"])) + ] + + # Row-oriented tables: metric names are columns and the requested + # model is a row (for example DeepSeek-VL2). + target_model = PdfExtractTool._select_positioned_target_model( + model_terms, + rows, + ) + target_base = re.sub(r"-(?:\d+(?:\.\d+)?[BMK])$", "", target_model, flags=re.I) + exact_model_rows = [ + (top, words) + for top, words in rows + if target_base + and PdfExtractTool._pdf_row_exact_term_match( + " ".join(str(word["text"]) for word in words), + target_base, + ) + ] + candidate_model_rows = exact_model_rows or [ + (top, words) + for top, words in rows + if target_base + and PdfExtractTool._pdf_row_contains_term( + " ".join(str(word["text"]) for word in words), + target_base, + ) + ] + for _top, words in candidate_model_rows: + row_text = " ".join(str(word["text"]) for word in words) + nums = _numbers(words) + if len(nums) >= len(metric_terms): + for metric, metric_x in metric_positions.items(): + nearest = min(nums, key=lambda item: abs(item[0] - metric_x)) + if abs(nearest[0] - metric_x) <= 30: + resolved[metric] = nearest[1] + if resolved: + break + + # A single row-oriented table can contain several models requested in + # the same query. Return a coordinate-joined record for each exact row + # instead of forcing the model to decode the remaining dense HTML row. + additional_row_resolutions: dict[str, dict[str, str]] = {} + for requested_model in dict.fromkeys(model_terms): + if requested_model == target_model: + continue + requested_base = re.sub( + r"-(?:\d+(?:\.\d+)?[BMK])$", "", requested_model, flags=re.I + ) + requested_rows = [ + words + for _top, words in rows + if requested_base + and PdfExtractTool._pdf_row_exact_term_match( + " ".join(str(word["text"]) for word in words), + requested_base, + ) + ] + for words in requested_rows: + nums = _numbers(words) + if len(nums) < len(metric_terms): + continue + values: dict[str, str] = {} + for metric, metric_x in metric_positions.items(): + nearest = min(nums, key=lambda item: abs(item[0] - metric_x)) + if abs(nearest[0] - metric_x) <= 30: + values[metric] = nearest[1] + if values: + additional_row_resolutions[requested_model] = values + break + + # Column-oriented tables: models are columns and metrics are rows (for + # example Qwen2.5-VL variants). Resolve a split size suffix such as + # "72B" to its x coordinate, then join each metric row at that x. + if len(resolved) < len(metric_terms) and target_model: + size_match = re.search(r"(\d+(?:\.\d+)?[BMK])$", target_model, re.I) + target_x = None + if size_match: + suffix = size_match.group(1).casefold() + base_hits = [ + (top, float(word["x0"])) + for top, words in rows for word in words + if target_base and str(word["text"]).casefold() == target_base.casefold() + ] + suffix_hits = [ + (top, float(word["x0"])) + for top, words in rows for word in words + if str(word["text"]).casefold() == suffix + ] + pairs = [ + (abs(base_top - suffix_top) + abs(base_x - suffix_x), suffix_x) + for base_top, base_x in base_hits for suffix_top, suffix_x in suffix_hits + if abs(base_top - suffix_top) <= 15 and abs(base_x - suffix_x) <= 30 + ] + if pairs: + target_x = min(pairs)[1] + if target_x is None: + normalized_target = normalize(target_model) + metric_tops = [ + top for top, words in rows + if any( + PdfExtractTool._pdf_words_contain_metric(words, metric) + for metric in metric_terms + ) + ] + first_metric_top = min(metric_tops) if metric_tops else float("inf") + column_fragments: dict[int, list[tuple[float, str]]] = {} + for top, words in rows: + if top >= first_metric_top: + continue + for word in words: + text = str(word["text"]) + if not re.search(r"[A-Za-z]", text): + continue + x_key = round(float(word["x0"])) + column_fragments.setdefault(x_key, []).append((top, text)) + matches: list[tuple[int, int]] = [] + for x_key, fragments in column_fragments.items(): + combined = " ".join(text for _top, text in sorted(fragments)) + normalized_combined = normalize(combined) + if ( + normalized_combined + and normalized_target + and ( + normalized_target in normalized_combined + or normalized_combined in normalized_target + ) + ): + matches.append((len(normalized_combined), x_key)) + if matches: + target_x = float(max(matches)[1]) + if target_x is None and target_model: + target_folded = target_model.casefold() + mode_terms = [] + if "instruct" in target_folded: + mode_terms.extend(["instruct", "non-thinking"]) + if "thinking" in target_folded and "instruct" not in target_folded: + mode_terms.append("thinking") + if mode_terms: + metric_tops = [ + top for top, words in rows + if any( + PdfExtractTool._pdf_words_contain_metric(words, metric) + for metric in metric_terms + ) + ] + first_metric_top = min(metric_tops) if metric_tops else float("inf") + mode_hits = [ + float(word["x0"]) + for top, words in rows + if top < first_metric_top + for word in words + if str(word["text"]).casefold() in mode_terms + ] + if mode_hits: + target_x = mode_hits[0] + if target_x is not None: + for metric in metric_terms: + for row_index, (_top, words) in enumerate(rows): + if not PdfExtractTool._pdf_words_contain_metric(words, metric): + continue + nums = _numbers(words) + if not nums: + adjacent_numeric_rows = [ + ( + abs(float(rows[candidate_index][0]) - float(_top)), + rows[candidate_index][1], + ) + for candidate_index in (row_index - 1, row_index + 1) + if 0 <= candidate_index < len(rows) + and abs(float(rows[candidate_index][0]) - float(_top)) <= 9 + and _numbers(rows[candidate_index][1]) + ] + if adjacent_numeric_rows: + nums = _numbers( + min(adjacent_numeric_rows, key=lambda item: item[0])[1] + ) + if nums: + nearest = min(nums, key=lambda item: abs(item[0] - target_x)) + if abs(nearest[0] - target_x) <= 30: + resolved[metric] = nearest[1] + break + + rendered = [] + for top, words in sorted(rows, key=lambda item: item[0]): + cells = " | ".join( + f"x={float(word['x0']):.0f}:{word['text']}" for word in words + ) + rendered.append(f"y={top:.0f} :: {cells}") + body = "\n".join(rendered) + if len(body) > MAX_OUTPUT_CHARS - 1000: + body = body[:MAX_OUTPUT_CHARS - 1000] + "\n[...positioned rows truncated]" + resolved_text = "" + all_resolutions: list[tuple[str, dict[str, str]]] = [] + if resolved: + all_resolutions.append((target_model or "requested model", resolved)) + all_resolutions.extend(additional_row_resolutions.items()) + resolved_sections: list[str] = [] + for resolved_model, resolved_values in all_resolutions: + missing_metrics = [ + metric for metric in metric_terms if metric not in resolved_values + ] + missing_note = ( + " | requested metrics not found: " + ", ".join(missing_metrics) + if missing_metrics else "" + ) + lock_values = { + metric: resolved_values[metric] + for metric in metric_terms + if metric in resolved_values + } + for metric in missing_metrics: + lock_values[metric] = None + resolved_sections.append( + "Resolved requested values by coordinate join: " + + resolved_model + + " | " + + " | ".join( + f"{metric}={resolved_values[metric]}" + for metric in metric_terms + if metric in resolved_values + ) + + missing_note + + "\n" + + "Resolved values JSON: " + + json.dumps({ + "model": resolved_model, + "values": lock_values, + "source": "pdf_extract_positioned_table", + "page": page_number, + }, sort_keys=True) + ) + if resolved_sections: + resolved_text = "\n".join(resolved_sections) + "\n" + return ( + f"[Positioned PDF table evidence, page {page_number}. Values sharing the same " + "x coordinate belong to the same column; map the requested model header x to " + "metric-row values at that x.]\n" + resolved_text + body + ) + + @staticmethod + def _local_text_evidence(path: Path, query: str) -> str: + """Return focused text pages when a local PDF has no positioned rows.""" + try: + import pdfplumber + except ImportError: + return "" + tokens = { + token.casefold() + for token in re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{2,}", query) + } + requested_labels = [ + re.sub(r"\s+", "", label.casefold()) + for label in re.findall(r"\b(?:table|figure|fig\.?)\s*\d+\b", query, re.I) + ] + pages: list[tuple[int, int, str]] = [] + page_columns: list[tuple[int, str, str]] = [] + try: + with pdfplumber.open(path) as pdf: + for page_number, page in enumerate(pdf.pages, 1): + text = str(page.extract_text(layout=True) or "").strip() + if not text: + continue + midpoint = float(page.width) / 2 + left = str( + page.crop((0, 0, midpoint, page.height)).extract_text() or "" + ).strip() + right = str( + page.crop((midpoint, 0, page.width, page.height)).extract_text() or "" + ).strip() + page_columns.append((page_number, left, right)) + folded = text.casefold() + compact = re.sub(r"\s+", "", folded) + label_hits = sum(label in compact for label in requested_labels) + numeric_count = len(re.findall(r"\b\d+(?:\.\d+)?\b", text)) + table_signal = bool( + re.search(r"\b(?:table|figure|fig\.?)\s*\d+\b", folded, re.I) + ) + result_table_signal = bool( + re.search( + r"(?:table\s*3|evaluation\s+results\s+on\s+groundingme|" + r"main\s+results|main\s+experimental\s+results)", + folded, + re.I, + ) + ) + appendix_table_signal = bool( + re.search( + r"(?:supplementary\s+material|detailed\s+subtask|" + r"table\s*[67])", + folded, + re.I, + ) + ) + requested_result_terms = { + "appendix", "average", "baseline", "experimental", + "leaderboard", "results", "score", "scores", "thinking", + } + result_table_boost = ( + 260 + if result_table_signal + and tokens & requested_result_terms + else 0 + ) + appendix_table_boost = ( + 220 + if appendix_table_signal + and ("appendix" in tokens or "thinking" in tokens) + else 0 + ) + score = ( + sum(token in folded for token in tokens) + + label_hits * 100 + + (8 if table_signal else 0) + + min(numeric_count, 20) // 4 + + result_table_boost + + appendix_table_boost + ) + pages.append((score, page_number, text)) + except Exception: + return "" + bibliography = PdfExtractTool._bibliography_evidence(page_columns, query) + if bibliography: + return bibliography + if not pages: + return "" + selected = sorted( + pages, key=lambda item: (item[0], -item[1]), reverse=True + )[:4] + # Put the most relevant pages first so the bounded output cap cannot + # truncate the requested table behind introductory pages. Page labels + # remain in each block, so callers can still cite the source page. + selected.sort(key=lambda item: (item[0], -item[1]), reverse=True) + body = "\n\n".join( + f"[Local PDF text evidence, page {page_number}]\n{text}" + for _score, page_number, text in selected + ) + if len(body) > MAX_OUTPUT_CHARS - 500: + body = body[:MAX_OUTPUT_CHARS - 500] + "\n[...local PDF text truncated]" + return body + + async def execute(self, content: str, ctx: dict) -> dict: + raw = content.strip() + try: + args = json.loads(raw) if raw.startswith("{") else {} + except json.JSONDecodeError: + args = {} + if not isinstance(args, dict): + args = {} + if not args: + lines = [line.strip() for line in raw.splitlines() if line.strip()] + args = { + "url": lines[0] if lines else "", + "query": ", ".join(lines[1:]), + } + url = str(args.get("url") or args.get("path") or "").strip() + query = str(args.get("query") or "").strip() + if not url: + return {"error": "pdf_extract: provide a PDF URL or local PDF path", "exit_code": 1} + if not query: + return { + "error": ( + "pdf_extract: provide query terms naming the target model, " + "metrics, or table so the returned evidence is precise" + ), + "exit_code": 1, + } + loop = asyncio.get_running_loop() + local_path = self._local_pdf_path(url) + if local_path is not None: + from src.tool_execution import _display_tool_path + + url = _display_tool_path(str(local_path)) + + async def _positioned_evidence() -> str: + try: + return await asyncio.wait_for( + loop.run_in_executor(None, self._positioned_table_evidence, url, query), + timeout=30, + ) + except Exception: + return "" + + # arXiv's PDF CDN can temporarily serve an older revision than its + # current HTML conversion. The HTML table is also substantially more + # legible than positioned PDF glyphs. Fetch both official forms in + # parallel and return complementary evidence, so a newly-added model + # row cannot disappear and trigger browser/shell retry loops. + html_url = self._arxiv_html_url(url) if local_path is None else "" + positioned_task = _positioned_evidence() + if html_url: + query_tokens = re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{1,}", query) + query_models = sorted( + { + token for token in query_tokens + if not self._looks_like_pdf_metric_term(token) + and self._looks_like_pdf_model_term(token) + }, + key=len, + reverse=True, + ) + # A broad query with two model names often locks onto the first of + # several identically numbered tables (for example dev Table 2 + # instead of test Table 2). Query each requested model separately; + # the rarer/longer row is emitted first and anchors the right table. + if len(query_models) > 1: + # Exact model-only focus keeps the matching row inside the + # passage budget. Repeating every metric acronym can rank the + # table introduction above a late model row and truncate the + # very values the caller requested. + html_queries = query_models + else: + html_queries = [query] + html_tasks = [ + WebFetchTool().execute( + json.dumps({"url": html_url, "query": focused, "full": True}), ctx + ) + for focused in html_queries + ] + gathered = await asyncio.gather(positioned_task, *html_tasks) + positioned = gathered[0] + html_outputs: list[str] = [] + for focused, html_result in zip(html_queries, gathered[1:]): + if html_result.get("exit_code") != 0: + continue + html_output = str(html_result.get("output") or "") + if html_output and html_output not in html_outputs: + html_outputs.append( + f"[Focused HTML evidence for: {focused}]\n{html_output}" + ) + evidence = [] + # Structured coordinate evidence is compact and column-safe. Put + # it first so a long prose extraction cannot consume the bounded + # response and truncate the exact table rows. + if positioned: + evidence.append(positioned) + if html_outputs: + evidence.append( + f"[Official arXiv HTML companion: {html_url}]\n" + + "\n\n--- additional requested model row ---\n\n".join(html_outputs) + ) + if evidence: + output = ( + f"Source: {url}\n\n" + "[Extraction guidance: align every value to the complete table header. " + "A row may contain extra unrequested generation metrics (for example VS or SSC); " + "skip those columns rather than treating the next contiguous number as the requested metric.]\n\n" + + "\n\n--- PDF/HTML corroboration ---\n\n".join(evidence) + ) + if len(output) > MAX_OUTPUT_CHARS: + output = output[:MAX_OUTPUT_CHARS] + "\n[...combined evidence truncated]" + return {"output": output, "exit_code": 0} + else: + positioned = await positioned_task + if positioned: + return {"output": f"Source: {url}\n\n{positioned}", "exit_code": 0} + if local_path is not None: + text_evidence = await asyncio.wait_for( + loop.run_in_executor( + None, self._local_text_evidence, local_path, query + ), + timeout=30, + ) + if text_evidence: + return { + "output": f"Source: {url}\n\n{text_evidence}", + "exit_code": 0, + } + return { + "error": ( + "pdf_extract: no selectable table/text evidence was found in the local PDF; " + "use inspect_media for scanned or visual pages" + ), + "exit_code": 1, + } + fetch_args = {"url": url, "query": query, "full": True} + result = await WebFetchTool().execute(json.dumps(fetch_args), ctx) + if result.get("exit_code") == 0: + result["output"] = ( + "[PDF extraction; the highest-ranked matching table window is below. " + "Align row values to the full column header, use only the requested model's " + "column, and do not re-query when the target row and header are present.]\n\n" + + str(result.get("output") or "") + ) + return result + + +class YouTubeTool: + """YouTube-specific read tool for videos, transcripts, comments, channels.""" + + _ACTIONS = {"comments", "transcript", "metadata", "latest_channel_video"} + + async def execute(self, content: str, ctx: dict) -> dict: + from services.youtube.youtube_handler import ( + extract_transcript_async, + fetch_youtube_comments, + init_youtube, + ) + + args, err = self._parse_args(content) + if err: + return {"error": err, "exit_code": 1} + action = str(args.get("action") or "metadata").strip().lower() + if action not in self._ACTIONS: + return { + "error": "youtube_tool: action must be one of " + + ", ".join(sorted(self._ACTIONS)), + "exit_code": 1, + } + + max_results = args.get("max_results") + if not isinstance(max_results, int) or max_results <= 0: + max_results = 20 + max_results = max(1, min(50, max_results)) + + progress_cb = ctx.get("progress_cb") if isinstance(ctx, dict) else None + if progress_cb: + await progress_cb({"elapsed_s": 0, "tail": f"youtube_tool: {action}"}) + + if action == "latest_channel_video": + channel = str(args.get("channel_url") or args.get("url") or args.get("handle") or "").strip() + if not channel: + return {"error": "youtube_tool latest_channel_video: provide channel_url or handle", "exit_code": 1} + return await self._latest_channel_video(channel, max_results=max_results) + + if action == "metadata" and not any(args.get(k) for k in ("url", "video_url", "video_id")): + channel = str(args.get("channel_url") or args.get("handle") or "").strip() + if channel: + return await self._latest_channel_video(channel, max_results=max_results) + + url_or_id = str(args.get("url") or args.get("video_url") or args.get("video_id") or "").strip() + # The shared extractor accepts ID prefixes inside text. At the tool + # boundary require the complete target, not a truncated invented ID. + if url_or_id.startswith(('http://', 'https://')): + parsed = urllib.parse.urlparse(url_or_id) + host = (parsed.hostname or '').lower() + if host in {'youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com'}: + parts = parsed.path.strip('/').split('/') + candidate = (urllib.parse.parse_qs(parsed.query).get('v', [''])[0] + if parsed.path == '/watch' else + parts[1] if len(parts) == 2 and parts[0] in {'shorts', 'embed', 'live'} else '') + elif host in {'youtu.be', 'www.youtu.be'}: + candidate = parsed.path.strip('/') + else: + candidate = '' + else: + candidate = url_or_id + if (not re.fullmatch(r'[A-Za-z0-9_-]{11}', candidate) + or (args.get('video_id') and str(args['video_id']) != candidate)): + return {"error": f"youtube_tool {action}: invalid video target. Resolve the actual video URL " + "from the user or an observed link. A title or channel page is not a video ID; " + "open the referenced video in the browser or use latest_channel_video first.", + "exit_code": 1, "failure_kind": "invalid_target"} + video_id = candidate + url = url_or_id if url_or_id.startswith(("http://", "https://")) else f"https://www.youtube.com/watch?v={video_id}" + + if action == "comments": + api_result = await self._comments_from_data_api(video_id, max_results) + if api_result.get("success"): + return {"output": self._format_comments(api_result, url), "exit_code": 0, "untrusted_content": True} + comments_data = await fetch_youtube_comments(video_id, max_comments=max_results, timeout=45) + if not comments_data.get("success"): + api_error = api_result.get("error") + fallback_error = comments_data.get("error") or "unknown error" + joined = f"{fallback_error}" + if api_error: + joined = f"YouTube Data API unavailable: {api_error}; yt-dlp fallback failed: {fallback_error}" + return {"error": f"youtube_tool comments: {joined}", "exit_code": 1, + "failure_kind": "comments_unavailable", "video_url": url, + "untrusted_content": True} + return {"output": self._format_comments(comments_data, url), "exit_code": 0, "untrusted_content": True} + + if action == "transcript": + init_youtube() + transcript_data = await extract_transcript_async(url, video_id) + if not transcript_data.get("success"): + return { + "error": f"youtube_tool transcript: {transcript_data.get('error') or 'transcript unavailable'}", + "exit_code": 1, + "untrusted_content": True, + } + return {"output": self._format_transcript(transcript_data, url), "exit_code": 0, "untrusted_content": True} + + return await self._metadata(url) + + def _parse_args(self, content: str) -> tuple[dict, str | None]: + raw = (content or "").strip() + if not raw: + return {}, "youtube_tool: provide JSON with action and url/video_id" + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return {"action": "metadata", "url": raw}, None + if not isinstance(parsed, dict): + return {}, "youtube_tool: arguments must be a JSON object" + return parsed, None + + async def _comments_from_data_api(self, video_id: str, max_results: int) -> dict: + api_key = os.getenv("YOUTUBE_API_KEY", "").strip() + if not api_key: + return {"success": False, "error": "YOUTUBE_API_KEY not set", "comments": []} + params = urllib.parse.urlencode({ + "part": "snippet", + "videoId": video_id, + "maxResults": str(max_results), + "order": "relevance", + "textFormat": "plainText", + "key": api_key, + }) + url = f"https://www.googleapis.com/youtube/v3/commentThreads?{params}" + + def _fetch() -> dict: + req = urllib.request.Request(url, headers={"User-Agent": "odysseus-ui/1.0"}) + with urllib.request.urlopen(req, timeout=15) as resp: + return json.loads(resp.read().decode("utf-8", errors="replace")) + + try: + data = await asyncio.wait_for(asyncio.to_thread(_fetch), timeout=20) + except Exception as e: + return {"success": False, "error": f"{type(e).__name__}: {e}", "comments": []} + + comments = [] + for item in data.get("items") or []: + snippet = ( + (item.get("snippet") or {}) + .get("topLevelComment", {}) + .get("snippet", {}) + ) + text = html.unescape(str(snippet.get("textDisplay") or snippet.get("textOriginal") or "")).strip() + if not text: + continue + comments.append({ + "author": snippet.get("authorDisplayName") or "Unknown", + "text": text, + "likes": snippet.get("likeCount") or 0, + }) + return {"success": True, "comments": comments, "count": len(comments), "source": "youtube_data_api"} + + def _format_comments(self, data: dict, url: str) -> str: + comments = [c for c in (data.get("comments") or []) if isinstance(c, dict)] + if not comments: + return f"YouTube comments for {url}: no comments returned." + lines = [ + f"YouTube comments for {url}", + f"Source: {data.get('source') or 'yt-dlp'}", + f"Count: {len(comments)}", + ] + title = str(data.get("title") or "").strip() + channel = str(data.get("channel") or "").strip() + if title: + lines.append(f"Title: {title}") + if channel: + lines.append(f"Channel: {channel}") + lines.append("") + for idx, comment in enumerate(comments, 1): + likes = comment.get("likes") or 0 + like_text = f" [{likes} likes]" if likes else "" + author = str(comment.get("author") or "Unknown").strip().lstrip("@") + text = re.sub(r"\s+", " ", str(comment.get("text") or "")).strip() + lines.append(f"{idx}. @{author}{like_text}: {text}") + output = "\n".join(lines) + return output[:MAX_OUTPUT_CHARS] + ("\n\n[...truncated]" if len(output) > MAX_OUTPUT_CHARS else "") + + def _format_transcript(self, data: dict, url: str) -> str: + lines = [ + f"YouTube transcript for {url}", + f"Video ID: {data.get('video_id') or ''}", + f"Language: {data.get('language') or 'unknown'}", + "", + ] + segments = data.get("segments") or [] + if segments: + for seg in segments: + if not isinstance(seg, dict): + continue + lines.append(f"[{seg.get('timestamp') or '??:??'}] {seg.get('text') or ''}") + else: + lines.append(str(data.get("transcript") or "")) + output = "\n".join(lines) + return output[:MAX_OUTPUT_CHARS] + ("\n\n[...truncated]" if len(output) > MAX_OUTPUT_CHARS else "") + + async def _metadata(self, url: str) -> dict: + result = await self._ytdlp_json(url, timeout=30) + if not result.get("success"): + return {"error": f"youtube_tool metadata: {result.get('error')}", "exit_code": 1, "untrusted_content": True} + data = result["data"] + lines = [ + f"Title: {data.get('title') or ''}", + f"Channel: {data.get('channel') or data.get('uploader') or ''}", + f"URL: {data.get('webpage_url') or url}", + f"Duration: {data.get('duration_string') or data.get('duration') or ''}", + f"View count: {data.get('view_count') or ''}", + f"Like count: {data.get('like_count') or ''}", + f"Upload date: {data.get('upload_date') or ''}", + ] + return {"output": "\n".join(lines), "exit_code": 0, "untrusted_content": True} + + async def _latest_channel_video(self, channel: str, *, max_results: int = 5) -> dict: + original_channel = channel + if channel.startswith("@"): + channel = f"https://www.youtube.com/{channel}/videos" + elif channel.startswith(("http://", "https://")): + if "/videos" not in urllib.parse.urlparse(channel).path: + channel = channel.rstrip("/") + "/videos" + else: + channel = f"https://www.youtube.com/{channel.strip('/')}/videos" + result = await self._ytdlp_json(channel, timeout=45, flat_playlist=True, playlist_end=max_results) + if not result.get("success"): + resolved_channel = await self._resolve_channel_from_search(original_channel) + if resolved_channel: + retry = await self._ytdlp_json( + resolved_channel, + timeout=45, + flat_playlist=True, + playlist_end=max_results, + ) + if retry.get("success"): + channel = resolved_channel + result = retry + else: + return { + "error": ( + "youtube_tool latest_channel_video: " + f"{result.get('error')}; resolved {resolved_channel} also failed: {retry.get('error')}" + ), + "exit_code": 1, + "untrusted_content": True, + } + else: + return {"error": f"youtube_tool latest_channel_video: {result.get('error')}", "exit_code": 1, "untrusted_content": True} + if not result.get("success"): + return {"error": f"youtube_tool latest_channel_video: {result.get('error')}", "exit_code": 1, "untrusted_content": True} + data = result["data"] + entries = [e for e in (data.get("entries") or []) if isinstance(e, dict)] + if not entries: + return {"error": "youtube_tool latest_channel_video: no videos found", "exit_code": 1, "untrusted_content": True} + entries = entries[:max_results] + heading = "Latest channel video" if len(entries) == 1 else f"Latest {len(entries)} channel videos" + lines = [f"{heading} for {channel}", ""] + for idx, entry in enumerate(entries, 1): + video_id = entry.get("id") or "" + video_url = entry.get("url") or entry.get("webpage_url") or "" + if video_id and not str(video_url).startswith("http"): + video_url = f"https://www.youtube.com/watch?v={video_id}" + lines.extend([ + f"{idx}. {entry.get('title') or ''}", + f" URL: {video_url}", + f" Duration: {entry.get('duration_string') or entry.get('duration') or ''}", + f" Video ID: {video_id}", + ]) + return {"output": "\n".join(lines), "exit_code": 0, "untrusted_content": True} + + async def _resolve_channel_from_search(self, channel: str) -> str: + query = self._channel_search_query(channel) + if not query: + return "" + result = await self._ytdlp_json( + f"ytsearch10:{query} official YouTube channel", + timeout=30, + flat_playlist=True, + playlist_end=10, + ) + if not result.get("success"): + return "" + entries = [e for e in (result.get("data") or {}).get("entries") or [] if isinstance(e, dict)] + if not entries: + return "" + target = self._normalize_channel_name(query) + + def _score(entry: dict) -> tuple[int, int]: + channel_name = self._normalize_channel_name(entry.get("channel") or entry.get("uploader") or "") + uploader_id = self._normalize_channel_name(entry.get("uploader_id") or "") + title = self._normalize_channel_name(entry.get("title") or "") + score = 0 + if channel_name == target: + score += 100 + elif target and target in channel_name: + score += 60 + if uploader_id == target: + score += 45 + elif target and target in uploader_id: + score += 25 + if target and target in title: + score += 10 + if re.search(r"\b(?:clips?|shorts?|two|second|fan|archive)\b", channel_name): + score -= 35 + if re.search(r"\b(?:clips?|shorts?|compilation|reacts?)\b", title): + score -= 10 + return score, int(entry.get("view_count") or 0) + + best = max(entries, key=_score) + if _score(best)[0] <= 0: + return "" + channel_url = str(best.get("channel_url") or "").strip() + uploader_url = str(best.get("uploader_url") or "").strip() + resolved = channel_url or uploader_url + if not resolved: + uploader_id = str(best.get("uploader_id") or "").strip() + if uploader_id.startswith("@"): + resolved = f"https://www.youtube.com/{uploader_id}" + if not resolved: + return "" + parsed = urllib.parse.urlparse(resolved) + if parsed.netloc and "/videos" not in parsed.path: + resolved = resolved.rstrip("/") + "/videos" + return resolved + + @staticmethod + def _channel_search_query(channel: str) -> str: + value = str(channel or "").strip() + if not value: + return "" + if value.startswith(("http://", "https://")): + path = urllib.parse.urlparse(value).path.strip("/") + parts = [part for part in path.split("/") if part and part.lower() != "videos"] + value = parts[-1] if parts else value + value = value.strip().lstrip("@").strip("/") + value = re.sub(r"(?i)^(?:c|channel|user)/", "", value) + value = re.sub(r"[_-]+", " ", value) + return re.sub(r"\s+", " ", value).strip() + + @staticmethod + def _normalize_channel_name(value: str) -> str: + value = str(value or "").lower().lstrip("@") + value = re.sub(r"[^a-z0-9]+", "", value) + return value + + async def _ytdlp_json(self, url: str, *, timeout: int, flat_playlist: bool = False, playlist_end: int = 5) -> dict: + binary = shutil.which("yt-dlp") + if not binary: + venv_binary = os.path.join(os.path.dirname(sys.executable), "yt-dlp") + binary = venv_binary if os.path.exists(venv_binary) else "" + if not binary: + return {"success": False, "error": "yt-dlp not installed"} + cmd = [binary, "--skip-download", "--no-warnings", "--js-runtimes", "node"] + if flat_playlist: + playlist_end = max(1, min(50, int(playlist_end or 5))) + cmd.extend(["--flat-playlist", "--playlist-end", str(playlist_end), "--dump-single-json"]) + else: + cmd.append("--dump-json") + cmd.append(url) + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except asyncio.TimeoutError: + with contextlib.suppress(Exception): + proc.kill() + await proc.wait() + return {"success": False, "error": f"yt-dlp timed out after {timeout}s"} + except Exception as e: + return {"success": False, "error": f"{type(e).__name__}: {e}"} + if proc.returncode != 0: + return {"success": False, "error": stderr.decode("utf-8", errors="replace")[:300]} + try: + data = self._parse_ytdlp_json_output(stdout.decode("utf-8", errors="replace")) + except json.JSONDecodeError as e: + return {"success": False, "error": f"invalid yt-dlp JSON: {e}"} + return {"success": True, "data": data} + + @staticmethod + def _parse_ytdlp_json_output(output: str) -> dict: + """Parse yt-dlp JSON from either a single object or JSON-lines output.""" + text = (output or "").strip() + try: + data = json.loads(text) + except json.JSONDecodeError: + entries = [] + for line in text.splitlines(): + line = line.strip() + if not line: + continue + parsed = json.loads(line) + if isinstance(parsed, dict): + entries.append(parsed) + return {"entries": entries} + if isinstance(data, dict): + return data + if isinstance(data, list): + return {"entries": [item for item in data if isinstance(item, dict)]} + return {"entries": []} + + +class PrivateBrowserTool: + """Small deterministic wrapper around Vercel's agent-browser CLI. + + This is intentionally narrower than handing the model the raw browser MCP + schema. Use web_search/web_fetch first; this exists for JS-rendered pages, + forms, clicks, screenshots, and logged-in browser state. + """ + + _ACTIONS = { + "open", + "read", + "snapshot", + "find", + "evaluate", + "click", + "fill", + "press", + "scroll", + "wait", + "screenshot", + "close", + "batch", + } + _AUTO_SCREENSHOT_ACTIONS = { + "open", + "snapshot", + "batch", + "click", + "fill", + "press", + "scroll", + } + + @staticmethod + def _shopping_landing_hint(output: str) -> str: + """Expose the actionable store link on global retail landing pages.""" + text = str(output or "") + if not re.search(r"\b(?:global|choose another store|store selector)\b", text, re.IGNORECASE): + return "" + match = re.search( + r'link\s+"(?P`` as normal answer deltas. ``state`` is a mutable per-round dict + carrying the emission bookkeeping; the stored answer is unchanged (see + ``visible_content_after_qwen_thinking``). + """ + events = [] + text = str(content or '') + if not text: + return events + lowered = text.casefold() + lead = len('') if lowered.startswith('') else 0 + close = lowered.find('') + if close < 0: + # The opening is injected by the chat template, so a progressive + # trace can arrive with no marker until . Emit the buffered text + # as thinking - holding back a possible partial close tag - so the trace + # streams live instead of dumping in one block at the end. + think_region = text[lead:max(lead, len(text) - len(''))] + answer_region = '' + else: + think_region = text[lead:close] + answer_region = text[close + len(''):] + emitted_think = int(state.get('think_emitted', 0) or 0) + if len(think_region) > emitted_think: + events.append({'delta': think_region[emitted_think:], 'thinking': True}) + state['think_emitted'] = len(think_region) + if close >= 0: + state['closed'] = True + emitted_answer = int(state.get('answer_emitted', 0) or 0) + if len(answer_region) > emitted_answer: + events.append({'delta': answer_region[emitted_answer:]}) + state['answer_emitted'] = len(answer_region) + return events + + def prior_short_answer_for_no_tool_summary(user_text, history): """Reuse the immediately prior concise answer for an explicit no-tool recap.""" text = str(user_text or '') @@ -2714,6 +2998,18 @@ def normalize_preview_entity_anchor_args(name, args): value = args.get(field) if isinstance(value, str) and value.startswith(prefix): args[field] = value[len(prefix):] + if canonical(name) == 'manage_notes': + # Some models copy the rendered note href without its leading '#'. + # Decode only a complete UUID; partial selectors retain the existing + # owner-bound resolution policy instead of gaining a new alias. + value = args.get('note_id') + if isinstance(value, str): + match = re.fullmatch( + r'note-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', + value, re.I, + ) + if match: + args['note_id'] = match.group(1) return args @@ -2754,6 +3050,9 @@ def normalize_preview_function_args(name, args, *, user_text=''): args.pop('filter', None) if canonical(name) == 'manage_notes': action = str(args.get('action') or '').strip().replace('-', '_').casefold() + if action == 'read' and (args.get('id') or args.get('note_id')): + args['action'] = 'view' + action = 'view' if action == 'list': request = str(user_text or '').casefold() # The notes backend interprets title/query/content on ``list`` as @@ -2770,6 +3069,10 @@ def normalize_preview_function_args(name, args, *, user_text=''): args.pop('archived', None) if not re.search(r'\b(?:checklists?|freeform\s+notes?|notes?\s+only)\b', request): args.pop('note_type', None) + query, _ = note_read_intent(user_text) + if query: + args['action'] = 'search' + args['query'] = query if canonical(name) == 'manage_skills': action = str(args.get('action') or '').strip().replace('-', '_').casefold() if action in {'update', 'change', 'revise'}: @@ -2897,8 +3200,9 @@ def scope_preview_contract(preview_contract, routed_contract, active_capabilitie requested_family_tools = set().union(*(FAMILY_TOOLS.get(f, ()) for f in active)) available_requested = routed_canonical & offered_canonical & requested_family_tools routed_canonical.update(canonical(name) for name in extra_tools) - if 'image_generation' not in active: - # A past image request must not keep generation warm on email/doc turns. + if 'image_generation' not in active and 'photo_editor' not in active: + # A past image request must not keep generation warm on email/doc turns, + # unless the photo_editor family (which bundles generate_image) is active. routed_canonical.discard('generate_image') missing_active = { f'capability:{family}' for family in active @@ -3230,7 +3534,11 @@ def _successful_email_identifiers(history): for message in history or (): if not isinstance(message, dict): continue - if message.get('role') == 'system' and 'Message UID:' in str(message.get('content') or ''): + active_reader = ( + message.get('_agent_injected') == 'context' + and (message.get('metadata') or {}).get('source') == 'active email reader' + ) + if (message.get('role') == 'system' or active_reader) and 'Message UID:' in str(message.get('content') or ''): extracted = _email_identifiers_from_text(message.get('content')) known['uid'].update(extracted['uid']) known['message_id'].update(extracted['message_id']) @@ -3518,6 +3826,20 @@ def evaluate_preview_call(name, args, user_text='', *, allow_execute_code=False, runtime_tools = PREVIEW_TOOLS | ( NATIVE_WORKSPACE_TOOLS if allow_native_workspace else frozenset() ) | ({canonical(tool) for tool in external_runtime_tools} if allow_native_workspace else set()) + # Spec turns are driven by the router: the Spec contract (its offered set) + # IS the model runtime, so a Spec pick is runnable. Keyed off the same + # marker the spec hook sets (`routing_experiment == 'spec-router'`), never + # a new flag, and never applied to compact/native turns. + try: + from src.turn_contract import active_turn_contract as _atc + _spec_contract = _atc() + if _spec_contract is not None and getattr( + _spec_contract, 'routing_experiment', '') == 'spec-router': + runtime_tools = runtime_tools | { + canonical(tool) for tool in getattr(_spec_contract, 'offered', ()) or () + } + except Exception: + pass if bare not in runtime_tools: return decision(False, 'tool_not_in_model_runtime') if bare in {canonical(tool) for tool in external_runtime_tools}: @@ -3559,6 +3881,9 @@ def evaluate_preview_call(name, args, user_text='', *, allow_execute_code=False, or bare in BROKERED_JOB_TOOLS or (contract_required and bare in {'edit_image', 'generate_image'}) or (contract_required and bare == 'create_document' and standalone_code_request(user_text)) + # The editor command is conservatively classified as a write; + # document.info is a read even when the user requested no edit. + or (bare == 'run_editor_command' and args.get('op') == 'document.info') ) ) if family not in authorized and not contextual_revision and not contract_scoped_mutation and not fixture_delete and not offered_private_action: @@ -3619,6 +3944,18 @@ def evaluate_preview_call(name, args, user_text='', *, allow_execute_code=False, # editor is offered only for an explicit image-editing turn. blocked_effects.remove(ToolEffect.NETWORK_EGRESS) allowed_effects.add(ToolEffect.NETWORK_EGRESS) + if ( + (contract_required or family in turn_authorized_families) + and bare in {'plan_edit', 'run_editor_command', 'get_editor_screenshot', 'ai_edit_image'} + ): + # Scoped photo_editor agent tools. They are never in the general tool + # list; the immutable turn contract only selects them for an explicit + # editor request (Quick Edit popup / image-edit turn), so their UI and + # brokered-network effects are covered by the contract seal. + blocked_effects.discard(ToolEffect.UI_SIDE_EFFECT) + allowed_effects.add(ToolEffect.UI_SIDE_EFFECT) + blocked_effects.discard(ToolEffect.NETWORK_EGRESS) + allowed_effects.add(ToolEffect.NETWORK_EGRESS) if ( contract_required and bare in {'download_model', 'serve_preset', 'stop_served_model'} @@ -3849,6 +4186,29 @@ def conversation(history_session, messages, *, owner=None, diagnostics=None): from src.turn_contract import result_reference_followup source_context = (recent_source_reference_context(groups[-2]) if len(groups) > 1 and result_reference_followup(current_text) else None) + # Tool output can dwarf the dialogue (especially a web search). Keep the + # visible exchange before enforcing the history cap, so one search-heavy + # turn cannot erase the subject of the next follow-up. + for index, group in enumerate(groups[:-1]): + if len(json.dumps(group)) <= 5000 or len(group) < 2: + continue + # Keep non-web tool evidence intact: a note/list follow-up may need + # exact result identifiers, and shortening it would be misleading. + if not any( + canonical((call.get('function') or {}).get('name', '')) in {'web_search', 'web_fetch'} + for message in group if message.get('role') == 'assistant' + for call in message.get('tool_calls') or () + ): + continue + answer = next((message.get('content') for message in reversed(group) + if message.get('role') == 'assistant' + and str(message.get('content') or '').strip()), None) + if answer is None: + continue + answer_text = str(answer) + if len(answer_text) > 3500: + answer_text = answer_text[:3500] + '\n[Earlier answer shortened for context.]' + groups[index] = [group[0], {'role': 'assistant', 'content': answer_text}] # Drop whole turns only, never orphan tool results from their native calls. groups = groups[-8:] while len(groups) > 1 and len(json.dumps(groups)) > 22000: @@ -4207,6 +4567,26 @@ def denied_response(): return 'I can’t perform that operation in this preview. No changes were made.' +WORKSPACE_REQUIRED_REFUSAL = ( + 'Shell and Python commands only run inside a workspace. ' + 'Open the code workspace (or set a working directory) and try again.' +) + + +def workspace_process_denial(executions) -> bool: + """True when a blocked call was refused because process execution needs a + sealed workspace (a plain chat turn carries none).""" + for entry in executions or (): + blob = ' '.join( + str(entry.get(key) or '') for key in ('error', 'output', 'failure_kind') + ) + # Resource identity also covers missing note and document selectors. + # Those denials must not turn into instructions to open a code workspace. + if 'sealed workspace' in blob.casefold(): + return True + return False + + def execution_has_write_effect(tool_name, content, capability, *, native_workspace_enabled): """Recognize successful native code that visibly mutates the workspace.""" @@ -4226,7 +4606,7 @@ _MUTATION_REQUEST = re.compile( r'\b(?:add|creat(?:e|ing)?|make|writ(?:e|ing)|sav(?:e|ing)|updat(?:e|ing)|edit(?:ing)?|chang(?:e|ing)|' r'set|schedul(?:e|ing)|reschedul(?:e|ing)|paus(?:e|ing)|resum(?:e|ing)|toggle|mark|' r'pin|archive|delet(?:e|ing)|remov(?:e|ing)|send|reply|draft|publish|run|launch|serve|start|stop|' - r'remember|remeber|forget|remind|review|proofread|suggest(?:ions?)?|expand|broaden|deepen|lighten|feedback|reserve|block)\b|' + r'remember|remeber|forget|remind|review|proofread|suggest(?:ions?)?|expand|broaden|deepen|lighten|reserve|block)\b|' r'\bgo\s+deeper\b', re.I, ) @@ -4262,7 +4642,7 @@ def mutation_action_requested(user_text): actionable = re.sub( r"\b(?:do\s+not|don't|never|without)\s+" r"(?:(?:chang(?:e|ing)|modif(?:y|ying)|edit(?:ing)?|delet(?:e|ing)|" - r"remov(?:e|ing)|send(?:ing)?|writ(?:e|ing)|creat(?:e|ing))\b)" + r"remov(?:e|ing)|send(?:ing)?|writ(?:e|ing)|creat(?:e|ing)|schedul(?:e|ing))\b)" r"[^.;\n]{0,120}?(?=(?:[.;\n]|\bbut\b|\binstead\b|$))", '', text, flags=re.I, ) @@ -5242,6 +5622,10 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac external_tool_schemas=None, temperature=0.0, context_resolution=None, request_authority=MISSING_AUTHORITY, **ignored): + if spec_driver_active(endpoint_url, model, owner, headers): + yield event({'delta': 'Spec driver mode is unavailable because its authority and completion checks are not supported. No action was performed.'}) + yield 'data: [DONE]\n\n' + return from src.generation_sampling import validate_temperature temperature = validate_temperature(temperature) # This path sends requests directly with httpx and therefore bypasses @@ -5622,6 +6006,38 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac 'multimodal_image_count': image_context_count, 'attachment_reference_count': attachment_refs, 'image_rehydration': conversation_diagnostics.get('image_rehydration')}) + _photo_editor_active = 'photo_editor' in (getattr(turn_contract, 'active_capabilities', ()) or ()) + _photo_editor_open = bool( + ((client_runtime_context or {}).get('web_ui_state') or {}).get('photo_editor_open') + ) + if _photo_editor_active and not _photo_editor_open: + _editor_guidance = ( + "This image isn't open in the editor yet, so I can't apply edits to the " + "canvas from chat. Open the attached image in the editor (Quick Edit) and " + "send this request there \u2014 I'll apply it to the canvas." + ) + history.append({'role': 'assistant', 'content': _editor_guidance}) + yield event({'delta': _editor_guidance}) + _action_prompt = str(direct_user_text or '') + for _annotation in ('\n\n[Image:', '\n[Image:', '\n\n[Attached', '\n[No vision model'): + _cut = _action_prompt.find(_annotation) + if _cut != -1: + _action_prompt = _action_prompt[:_cut] + _action_prompt = _action_prompt.strip() or str(direct_user_text or '').strip() + yield event({ + 'type': 'final_response', + 'content': _editor_guidance, + 'photo_editor_action': { + 'kind': 'open_and_apply', + 'prompt': _action_prompt, + }, + }) + yield event({'type': 'metrics', 'data': { + 'model': model, 'rounds': 0, 'round_texts': [_editor_guidance], + 'tool_events': [], 'tool_calls': 0, 'agent_rounds': 0, + }}) + yield 'data: [DONE]\n\n' + return try: async with httpx.AsyncClient( timeout=preview_http_timeout( @@ -5703,6 +6119,40 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac usage_in += intent_accounting.get('input_tokens', 0) usage_out += intent_accounting.get('output_tokens', 0) has_real_usage = intent_accounting.get('usage_source') == 'real' + # Optional env-gated router decision (spec surface only). Computed + # once per user turn; disabled/None leaves behaviour unchanged. + _spec_route_think = await spec_router_thinking_decision_async( + endpoint_url, model, owner, headers, direct_user_text) + if _spec_route_think is not None: + logging.getLogger(__name__).info( + 'spec think router: think=%s p_think=%s ms=%s', + _spec_route_think.get('think'), + _spec_route_think.get('p_think'), + _spec_route_think.get('ms')) + # Optional env-gated direct-answer override (spec surface only): + # when the spec router chose answer_directly with high confidence, + # round 1 is sent with tool_choice 'none' (same tools list). + _spec_direct_p = spec_direct_no_tools_p( + direct_user_text, endpoint_url, model, owner, headers) + if (turn_contract.required or turn_contract.unavailable + or turn_contract.required_read_operation is not None): + _spec_direct_p = None + # Option B: send round 1 with no tools at all for a direct answer. + _spec_direct_empty_menu = ( + _spec_direct_p is not None and spec_direct_mode() == 'empty_menu') + # ODYSSEUS_SPEC_MODE=driver: the router (not the 9B) picks the tool + # every round; the 9B only writes the arguments or the answer. + # Driver execution is refused at entry. Keep this false even if + # configuration changes while the stream is already in progress. + _spec_driver_on = False + # Driver mode narrows the contract per round to the Spec pick. + # Rebuild each round from this frozen base so the narrowing does + # not accumulate/mutate across rounds. + _driver_base_contract = turn_contract + # Per-round Spec pick (driver mode): 'answer_directly' | 'ask_user' + # | 'tool' | None (router fallback). Set inside the round loop. + _driver_round_kind = None + _driver_round_pick = None # One extra iteration is available only when a provider emits raw # tool markup during the normal final no-tools round. Ordinary # turns still obey ``round_limit`` exactly. @@ -5839,7 +6289,9 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac 'completion_reserve_seconds': artifact_completion_reserve_seconds, }) request_messages = provider_request_messages( - prune_multimodal_images(history, max_images=3) + prune_multimodal_images( + keep_recent_image_turns(history, keep=2), max_images=3 + ) ) if prior_summary_answer or force_no_tools_next_round or reserve_final_synthesis: round_offered = [] @@ -6036,6 +6488,94 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac break request['tools'] = source_tools request['tool_choice'] = 'required' + # ODYSSEUS_SPEC_MODE=driver: the router is the tool driver. + # Ask it for this round's single next action (any confidence); + # the 9B then only writes the arguments (a named tool_choice) or + # the answer (tools key omitted; vLLM rejects an empty array). + # Router error/timeout keeps the normal menu for this round. + if _spec_driver_on and round_offered: + # The router chooses from the FULL Ajax Spec surface (~all + # built-ins minus disabled), not the compact round menu, so + # a Spec pick is never silently out of reach. The 9B then + # only writes the arguments (a named tool_choice) or the + # answer (tools key omitted; vLLM rejects an empty array). + # Router error/timeout keeps the normal menu this round. + from src.tool_schemas import FUNCTION_TOOL_SCHEMAS as _SPEC_SURFACE + _drv = spec_driver_pick(history, disabled_tools, direct_user_text) + if _drv is not None: + _drv_choice = _drv.get('choice') + if _drv_choice == 'answer_directly': + _driver_round_kind = 'answer_directly' + request.pop('tools', None) + request.pop('tool_choice', None) + elif _drv_choice == 'ask_user': + _driver_round_kind = 'ask_user' + _ask = [s for s in _SPEC_SURFACE + if (s.get('function') or {}).get('name') == 'ask_user'] + if _ask: + request['tools'] = [_ask[0]] + request['tool_choice'] = { + 'type': 'function', + 'function': {'name': 'ask_user'}, + } + if _ask[0] not in round_offered: + round_offered = list(round_offered) + [_ask[0]] + elif _drv_choice == 'tool' and _drv.get('tool_name'): + _name = _drv['tool_name'] + _sel = [s for s in list(round_offered) + list(_SPEC_SURFACE) + if (s.get('function') or {}).get('name') == _name] + if _sel: + _driver_round_kind = 'tool' + _driver_round_pick = _name + request['tools'] = [_sel[0]] + request['tool_choice'] = { + 'type': 'function', + 'function': {'name': _name}, + } + # The per-call validator looks the schema up in + # `round_offered` (not `request['tools']`), so the + # Spec pick must be present there too. + round_offered = [_sel[0]] + # Driver mode narrows the whole contract to the + # Spec pick for this round: the router is the + # authority, and the executor must be able to run + # the picked tool. Build a valid contract (the + # TurnContract invariant requires + # required <= offered <= executable and + # schema_json names == offered). + try: + from src.turn_contract import ( + _ACTIVE_CONTRACT as _TC, + canonical_tool as _ct, + ) + _pick = _ct(_name) + turn_contract = replace( + _driver_base_contract, + required=frozenset(), + offered=frozenset({_pick}), + executable=frozenset( + set(_driver_base_contract.executable) | {_pick}), + schema_json=(json.dumps(_sel[0]),), + required_read_operation=None, + ) + _TC.set(turn_contract) + except Exception: + turn_contract = _driver_base_contract + logging.getLogger(__name__).info( + '[spec-driver] round=%s pick=%s contract admit failed; ' + 'normal contract kept', round_number, _name) + else: + logging.getLogger(__name__).info( + '[spec-driver] round=%s pick=%s not executable; normal menu this round', + round_number, _name) + logging.getLogger(__name__).info( + '[spec-driver] round=%s pick=%s p=%s', round_number, + (_drv.get('tool_name') if _drv_choice == 'tool' + else _drv_choice), + _drv.get('p')) + else: + logging.getLogger(__name__).info( + '[spec-driver] round=%s pick=fallback p=None', round_number) pending, content, round_reasoning = {}, '', '' document_preview_index = None document_preview_content = '' @@ -6056,6 +6596,49 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac request = search_tool_choice_request(request) tool_call_requested = request.get('tool_choice') not in (None, 'auto', 'none') request = provider_compatible_tool_choice_request(request, model) + # Log the tool surface actually sent to the provider: the + # metrics `tool_schema_names` reports the pre-narrowing + # `offered` set, which hid the fact that a forced choice had + # shrunk the provider tools array to one schema. + logging.getLogger(__name__).info( + "clean preview round %s sending %d tool(s): %s", + round_number, + len(request.get('tools') or []), + [ + (schema.get('function') or {}).get('name') + for schema in (request.get('tools') or []) + ][:40], + ) + if round_number == 1 and _spec_direct_p is not None: + # The router decided to answer directly with high + # confidence. + if _spec_direct_empty_menu: + # Option B: an empty menu — send no tools at all, so + # nothing can be executed and no markup is recovered. + request['tools'] = [] + request.pop('tool_choice', None) + tool_call_requested = False + # Keep the thinking router's decision; never let the + # no-tools progressive rule turn thinking on when the + # router said off. + if not (_spec_route_think or {}).get('think'): + ctk = dict(request.get('chat_template_kwargs') or {}) + ctk['enable_thinking'] = False + request['chat_template_kwargs'] = ctk + logging.getLogger(__name__).info( + '[spec-harness] direct answer: empty menu p=%s', + _spec_direct_p) + else: + # Default: keep the same tools list (prefix-cache + # friendly) but forbid a tool call this round. Thinking + # is left to the thinking router; offered stays + # non-empty so this does not switch progressive + # thinking on. + request['tool_choice'] = 'none' + tool_call_requested = False + logging.getLogger(__name__).info( + '[spec-harness] direct answer: tool_choice none p=%s', + _spec_direct_p) if active_editor_target and request.get('tools'): # Edits share mutable document state. Await a saved result # before asking for another call; one call may still batch @@ -6144,6 +6727,7 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac and not progressive_thinking and not source_lookup_pending and not tool_call_requested + and not _spec_direct_empty_menu ): text_event = {'delta': text} if replace_streamed_draft_on_finish and not streamed_round_text: @@ -6197,6 +6781,13 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac if content and not prior_summary_answer and not source_lookup_pending: yield event({'delta': content}) streamed_round_text = True + if _spec_direct_empty_menu and pending: + # Option B: nothing is offered, so never execute a tool + # call the model still produced (native shape). + logging.getLogger(__name__).info( + '[spec-harness] direct answer: dropped %d tool call(s)', + len(pending)) + pending = {} proposed = [pending[i] for i in sorted(pending)] # A lead-in emitted before a tool call is live progress, not # part of the terminal answer. Replace that draft when the @@ -6204,7 +6795,9 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac if proposed and streamed_round_text: replace_streamed_draft_on_finish = True unexecutable_dsml_completion = False - if not proposed and 'DSML' in content: + if (not proposed and not _spec_direct_empty_menu + and _driver_round_kind != 'answer_directly' + and 'DSML' in content): offered_by_canonical = { canonical(schema['function']['name']): schema['function']['name'] for schema in round_offered @@ -6252,6 +6845,15 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac }) else: unexecutable_dsml_completion = True + if _spec_direct_empty_menu and content: + # Option B: show the answer text without any tool-call + # markup the model may still have written (nothing offered). + _stripped = re.sub( + r']*>.*?', '', + content, flags=re.DOTALL | re.IGNORECASE).strip() + if _stripped != content.strip(): + content = _stripped + replace_streamed_draft_on_finish = True if source_lookup_pending and not proposed: if source_answer_retries < 1 and round_number < round_limit: source_answer_retries += 1 @@ -6276,6 +6878,37 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac ) if active_editor_target: proposed = drop_redundant_editor_noops(proposed) + # ODYSSEUS_SPEC_MODE=driver: Spec is the tool driver, so execute + # ONLY the call Spec picked this round. Drop (never execute) any + # other native call and any text/DSML-recovered call; when Spec + # said answer_directly, keep no calls and strip tool markup so + # the remaining text is the answer. + if _driver_round_kind == 'answer_directly': + if proposed: + for _c in proposed: + logging.getLogger(__name__).info( + '[spec-driver] dropped off-pick call name=%s', + (_c.get('function') or {}).get('name')) + proposed = [] + if content: + _stripped = strip_tool_blocks(content, skip_fenced=True).strip() + _stripped = re.sub( + r']*>.*?', '', + _stripped, flags=re.DOTALL | re.IGNORECASE).strip() + if _stripped != content: + content = _stripped + replace_streamed_draft_on_finish = True + elif _driver_round_kind in ('tool', 'ask_user'): + _want = 'ask_user' if _driver_round_kind == 'ask_user' else _driver_round_pick + _keep = [] + for _c in proposed: + _cn = (_c.get('function') or {}).get('name') + if canonical(str(_cn or '')) == canonical(str(_want or '')): + _keep.append(_c) + else: + logging.getLogger(__name__).info( + '[spec-driver] dropped off-pick call name=%s', _cn) + proposed = _keep if model_choice_experiment: for proposal in proposed: yield event({'type': 'model_tool_proposal', 'round': round_number, @@ -6695,6 +7328,25 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac # that draft and persisted history with the evidence-backed result. history.pop() refusal = denied_response() + # A photo-editor turn with no canvas command actually executed + # (no browser editor attached) cannot claim the edit. Point the + # user at the editor instead of the generic no-op refusal. + if 'photo_editor' in (getattr(turn_contract, 'active_capabilities', ()) or ()): + touched_canvas = any( + str(e.get('tool') or '') in { + 'run_editor_command', 'get_editor_screenshot', + 'ai_edit_image', 'generate_image', + } + and not e.get('error') and e.get('exit_code', 0) == 0 + for e in executions + ) + if not touched_canvas: + refusal = ( + 'This photo is not open on the editor canvas, so I cannot ' + 'apply edits to it from chat. Open the attached image in the ' + 'editor (Quick Edit) and send the request there, and I will ' + 'apply it to the canvas.' + ) history.append({'role': 'assistant', 'content': refusal}) yield event({'type': 'final_response', 'content': refusal}) break @@ -7948,6 +8600,15 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac output, user_text=latest_user, max_items=contract_item_limit(turn_contract, 20), ) + followup = note_locator_followup(output, user_text=latest_user) + if followup: + if round_number < round_limit: + structured_terminal_response = '' + round_recovery_messages.append(followup) + else: + structured_terminal_response = ( + 'I found a note, but could not read its content in this turn.' + ) action = str(args.get('action') or '').replace('-', '_').casefold() if ( action in {'search', 'find'} diff --git a/src/context_compactor.py b/src/context_compactor.py index 05b53ce95..09e2e59f6 100644 --- a/src/context_compactor.py +++ b/src/context_compactor.py @@ -92,6 +92,84 @@ def prune_multimodal_images( return pruned +_IMAGE_TURN_LABEL_RE = re.compile(r"\[Image(?:s)?(?:[ \t]{1,64}attached)?:[ \t]{0,64}([^\]]{1,512})\]", re.IGNORECASE) + + +def _image_turn_label(message: Dict) -> str: + """Best-effort '[Image: name]' label for a message whose image parts are dropped.""" + name = "" + content = message.get("content") if isinstance(message, dict) else None + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and item.get("type") in _MULTIMODAL_IMAGE_TYPES: + image_url = item.get("image_url") + candidate = item.get("name") or (image_url.get("name") if isinstance(image_url, dict) else None) + if candidate: + name = str(candidate) + break + if not name: + text = _content_as_text(content) + match = _IMAGE_TURN_LABEL_RE.search(text) + if match: + name = match.group(1).strip() + return f"[Image: {name}]" if name else "[Image]" + + +def keep_recent_image_turns(messages: List[Dict], *, keep: int = 2) -> List[Dict]: + """Keep image parts only for the last `keep` messages that carry an image. + + Older image blocks are dropped and replaced with a short ``[Image: name]`` + text label so the model still knows a picture was shared (and its name) + without paying the image tokens on every later turn. Non-text, non-image + multimodal blocks (e.g. audio) are left untouched. + """ + limit = max(0, int(keep)) + image_messages = [ + index for index, message in enumerate(messages) + if isinstance(message, dict) + and isinstance(message.get("content"), list) + and any( + isinstance(item, dict) and item.get("type") in _MULTIMODAL_IMAGE_TYPES + for item in message["content"] + ) + ] + if len(image_messages) <= limit: + return list(messages) + selected = set(image_messages[len(image_messages) - limit:]) if limit else set() + + trimmed: List[Dict] = [] + for message_index, message in enumerate(messages): + if message_index in selected or not isinstance(message, dict): + trimmed.append(message) + continue + content = message.get("content") + if not isinstance(content, list): + trimmed.append(message) + continue + kept_blocks = [ + dict(item) if isinstance(item, dict) else item for item in content + if not (isinstance(item, dict) and item.get("type") in _MULTIMODAL_IMAGE_TYPES) + ] + if len(kept_blocks) == len(content): + trimmed.append(message) + continue + label = _image_turn_label(message) + # Attach the label to the trailing text block when possible so we do not + # introduce an extra content block; otherwise append a fresh text block. + labelled = False + for item in reversed(kept_blocks): + if isinstance(item, dict) and item.get("type") == "text": + item["text"] = (item.get("text") or "").rstrip() + f"\n{label}" + labelled = True + break + if not labelled: + kept_blocks.append({"type": "text", "text": label}) + cloned = dict(message) + cloned["content"] = kept_blocks + trimmed.append(cloned) + return trimmed + + COMPACT_THRESHOLD = 0.85 # Trigger compaction at 85% of context window SUMMARY_MAX_TOKENS = 1024 SMALL_CONTEXT_LIMIT = 8192 # Models with context <= this get aggressive trimming @@ -106,7 +184,7 @@ def auto_compact_threshold_percent() -> int: return max(50, min(95, value)) # Cursor-style self-summarization prompt — produces structured, dense summaries -SELF_SUMMARY_SYSTEM_PROMPT = """You are summarizing a conversation to preserve context after compaction. Produce a structured summary that lets the conversation continue seamlessly. +SELF_SUMMARY_SYSTEM_PROMPT = """You are summarizing a conversation with Ajax, the user's AI assistant, to preserve context after compaction. Produce a structured summary that lets Ajax continue seamlessly. Ajax is the assistant's name; Odysseus is the app name. Use this format: diff --git a/src/document_processor.py b/src/document_processor.py index 38643cb5a..a623c1a1b 100644 --- a/src/document_processor.py +++ b/src/document_processor.py @@ -2,6 +2,7 @@ """Document processing: PDF/OCR extraction, text file handling, image VL analysis, user content building.""" import os +import io import logging import mimetypes import base64 @@ -496,6 +497,49 @@ def analyze_image_with_vl(image_path: str, owner: str | None = None) -> str: return analyze_image_with_vl_result(image_path, owner=owner).get("text", "") +def _vision_max_side() -> int: + """Longest-side cap for images inlined into the chat request (tokens).""" + try: + value = int(os.getenv("ODYSSEUS_VISION_MAX_SIDE", "1280") or 1280) + except (TypeError, ValueError): + value = 1280 + return max(0, value) + + +def encode_image_data_url(path: str, fallback_format: str) -> tuple[str, str]: + """Return (data_url, image_format) for an image, downscaled if large. + + Downscale the longest side to ODYSSEUS_VISION_MAX_SIDE (default 1280) and + re-encode as JPEG so the chat model does not pay full-resolution image + tokens. Small/undecodable images fall back to the original bytes so a + real attachment is never dropped. + """ + max_side = _vision_max_side() + try: + from PIL import Image # local import: keep module import cost low + with Image.open(path) as img: + img = img.convert("RGB") if img.mode not in ("RGB", "L") else img + width, height = img.size + if max_side and max(width, height) > max_side: + scale = max_side / float(max(width, height)) + img = img.resize( + (max(1, round(width * scale)), max(1, round(height * scale))), + Image.LANCZOS, + ) + buffer = io.BytesIO() + img.save(buffer, format="JPEG", quality=85, optimize=True) + return ( + "data:image/jpeg;base64," + base64.b64encode(buffer.getvalue()).decode("utf-8"), + "jpeg", + ) + except Exception as exc: + logger.debug(f"Image downscale failed for {path}: {exc}; sending original bytes") + with open(path, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode("utf-8") + image_format = (fallback_format or "png").lower() + return f"data:image/{image_format};base64,{encoded_string}", image_format + + def build_user_content( text: str, attachment_ids: list[str] | None, @@ -542,15 +586,14 @@ def build_user_content( if upload_handler.is_image_file(display_name, mime): try: - with open(path, "rb") as image_file: - encoded_string = base64.b64encode(image_file.read()).decode("utf-8") # Extensionless uploads (e.g. a pasted screenshot) have no ext, # so fall back to the resolved MIME subtype rather than emitting # an invalid "data:image/;base64," with an empty subtype. - image_format = ext[1:] or (mime.split("/", 1)[1] if mime.startswith("image/") else "png") + fallback_format = ext[1:] or (mime.split("/", 1)[1] if mime.startswith("image/") else "png") + data_url, image_format = encode_image_data_url(path, fallback_format) content.append({ "type": "image_url", - "image_url": {"url": f"data:image/{image_format};base64,{encoded_string}"}, + "image_url": {"url": data_url}, }) except Exception as e: logger.error(f"Failed to encode image {fid}: {e}") diff --git a/src/editor_agent_model.py b/src/editor_agent_model.py new file mode 100644 index 000000000..7c9bdf98a --- /dev/null +++ b/src/editor_agent_model.py @@ -0,0 +1,732 @@ +"""Editor agent/planner model resolution. + +The image-editor Quick Edit planner (`/api/editor/plan`, `/api/editor/agent`) +runs on an explicitly configured local vision model when one is reachable, +with the configured chat model as a fallback. This module owns: + + * the local model base URLs / model id (settings-overridable), + * an ordered candidate list (per-request ``agent_model`` override, then the + local Qwen3.5-9B, then the hosted default/utility chat model), + * a vision fallback chain: local model WITH the canvas screenshot, else a + text-only call whose image is described by a separate VL caption model, + * a lightweight reachability probe for the configured local bases. + +Everything is best-effort: when no local server is reachable the planner +transparently falls back to the previously-used hosted chat model, so the +editor agent never regresses to "no plan". +""" + +from __future__ import annotations + +import contextvars +import json +import logging +import os +import re +import threading +from typing import Any, Dict, List, Optional, Tuple + +import httpx + +logger = logging.getLogger(__name__) + +# Local endpoints are opt-in through settings or ODYSSEUS_EDITOR_AGENT_BASES. +DEFAULT_LOCAL_BASES = [] +DEFAULT_LOCAL_MODEL = "Qwen/Qwen3.5-9B" + +# Vision caption model used when the editor-agent model cannot accept images. +DEFAULT_CAPTION_BASE = "" +DEFAULT_CAPTION_MODEL = "Qwen/Qwen3.5-9B" + +# Served-model-id cache per base (populated on first use). The local servers 400 +# with "Failed to load model" when asked for an id they do not serve (e.g. the +# client's default "Qwen3.5-9B-MLX-8bit", or a stale caption model), so we +# resolve the requested id against GET {base}/models and fall back to one that +# is actually served. +_MODELS_CACHE: Dict[str, set] = {} + +_CAPTION_INSTRUCTION = ( + "Describe this image in one or two sentences for a photo editor: the main " + "subject, the background/scene, lighting, dominant colours, and any obvious " + "flaws (blown highlights, colour cast, clutter, blur)." +) + + +def _setting(key: str, default: Any = "") -> Any: + try: + from src.settings import load_settings + + value = load_settings().get(key) + return value if value not in (None, "") else default + except Exception: + return default + + +def local_bases() -> List[str]: + override = _setting("editor_agent_base_url") + if override: + return [str(override)] + env = os.getenv("ODYSSEUS_EDITOR_AGENT_BASES") + if env: + bases = [b.strip() for b in env.split(",") if b.strip()] + if bases: + return bases + return list(DEFAULT_LOCAL_BASES) + + +def local_model() -> str: + return str(_setting("editor_agent_model", DEFAULT_LOCAL_MODEL)) + + +def caption_base() -> str: + return str(_setting("editor_caption_base_url", DEFAULT_CAPTION_BASE)) + + +def caption_model() -> str: + return str(_setting("editor_caption_model", DEFAULT_CAPTION_MODEL)) + + +# Per-request / per-plan context (contextvars keep this correct under the +# async event loop without sharing state between concurrent editor turns). +_override: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( + "editor_agent_model_override", default=None +) +_last_model: contextvars.ContextVar[Optional[Tuple[str, str]]] = contextvars.ContextVar( + "editor_agent_last_model", default=None +) + + +def set_override(value: Optional[str]) -> None: + _override.set(value or None) + + +def get_override() -> Optional[str]: + return _override.get() + + +def set_last_model(model: str, source: str) -> None: + _last_model.set((model, source)) + + +def get_last_model() -> Optional[Tuple[str, str]]: + return _last_model.get() + + +def _chat_url(base: str) -> str: + b = str(base or "").rstrip("/") + if b.endswith("/chat/completions"): + return b + return b + "/chat/completions" + + +def _split_override(override: str) -> Tuple[str, str]: + """Parse ``base::model`` (or a bare model id).""" + raw = str(override or "").strip() + if "::" in raw: + base, model = raw.split("::", 1) + return base.strip(), model.strip() + return "", raw + + +def _has_image(messages: List[dict]) -> bool: + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "image_url": + return True + return False + + +def _first_image(messages: List[dict]) -> Optional[str]: + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "image_url": + url = (part.get("image_url") or {}).get("url") + if isinstance(url, str): + return url + return None + + +def _strip_images(messages: List[dict]) -> List[dict]: + out: List[dict] = [] + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + text = "\n".join( + str(part.get("text") or "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ) + out.append({"role": msg.get("role", "user"), "content": text}) + else: + out.append({"role": msg.get("role", "user"), "content": content}) + return out + + +def _with_caption(messages: List[dict], caption: str) -> List[dict]: + note = f"\n\n[Canvas screenshot caption: {caption}]" + out = [dict(m) for m in messages] + for msg in reversed(out): + if msg.get("role") == "user": + msg["content"] = str(msg.get("content") or "") + note + break + return out + + +async def _post_chat( + base: str, + model: str, + messages: List[dict], + headers: Optional[dict], + timeout: float, + max_tokens: int = 1024, + response_format: Optional[dict] = None, +) -> Optional[str]: + if not base or not model: + return None + payload: Dict[str, Any] = { + "model": model, + "messages": messages, + "temperature": 0.2, + "max_tokens": max_tokens, + "stream": False, + } + # Constrain the planner to a strict JSON ops schema (structured outputs / + # grammar) so the model can only emit valid editor ops. Unknown servers + # ignore the field; the server-side schema validation is the real enforcer. + if isinstance(response_format, dict): + payload["response_format"] = response_format + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post(_chat_url(base), json=payload, headers=headers or {}) + if resp.status_code != 200: + logger.debug( + "editor agent chat %s -> %s (%s)", base, resp.status_code, resp.text[:200] + ) + return None + data = resp.json() + except Exception as exc: + logger.debug("editor agent chat %s failed: %s", base, exc) + return None + try: + return data["choices"][0]["message"]["content"] + except Exception: + return None + + +async def _served_models(base: str, timeout: float = 3.0) -> set: + """Return the set of model ids a base serves (cached; empty on failure).""" + if not base: + return set() + cached = _MODELS_CACHE.get(base) + if cached is not None: + return cached + ids: set = set() + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.get(base.rstrip("/") + "/models") + if resp.status_code == 200: + ids = { + str(m.get("id")) + for m in (resp.json().get("data") or []) + if isinstance(m, dict) and m.get("id") + } + except Exception: + ids = set() + _MODELS_CACHE[base] = ids + return ids + + +async def _resolve_model(base: str, requested: str, fallback: str) -> str: + """Return a model id ``base`` actually serves. + + If the requested id is served, keep it; otherwise use ``fallback`` (when + served) or any served id. When the served list is unknown (server down), + keep the requested id (or ``fallback``). + """ + requested = str(requested or "").strip() + fallback = str(fallback or "").strip() + ids = await _served_models(base) + if ids: + if requested and requested in ids: + return requested + if fallback in ids: + return fallback + return sorted(ids)[0] + return requested or fallback + + +async def caption_image(data_url: str, owner: Optional[str] = None, timeout: float = 90.0) -> Optional[str]: + if not isinstance(data_url, str) or not data_url.startswith("data:image/"): + return None + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": _CAPTION_INSTRUCTION}, + {"type": "image_url", "image_url": {"url": data_url}}, + ], + } + ] + base = caption_base() + if not base: + return None + model = await _resolve_model(base, caption_model(), local_model()) + return await _post_chat(base, model, messages, None, timeout, max_tokens=200) + + +_DONE_RE = re.compile(r"^\s*DONE\b", re.I) +_RETRY_RE = re.compile(r"RETRY:?\s*(.+)", re.I | re.S) + + +def _parse_self_check(text: str) -> Dict[str, Any]: + s = str(text or "").strip() + if _DONE_RE.match(s): + return {"done": True, "issues": [], "retry_hint": "", "text": s} + m = _RETRY_RE.search(s) + if m or re.search(r"\bretry\b", s, re.I): + hint = (m.group(1).strip() if m else s) or s + return {"done": False, "issues": [hint], "retry_hint": hint, "text": s} + return {"done": True, "issues": [], "retry_hint": "", "text": s} + + +async def self_check( + before: Any, + after: Any, + prompt: str, + model: Optional[str] = None, + owner: Optional[str] = None, + timeout: float = 30.0, +) -> Dict[str, Any]: + """Same-origin VL self-check: compare BEFORE vs AFTER against the intent. + + The browser cannot call the local VL server directly (it sends no CORS + headers), so ``/api/editor/self-check`` proxies here. Returns + ``{done, issues, retry_hint, text}``; on any failure (bad input, the model + is unreachable, or a timeout) it degrades to ``done=True`` with + ``reason="unavailable"`` so the client never gets stuck retrying. + """ + if not ( + isinstance(before, str) and before.startswith("data:image/") + and isinstance(after, str) and after.startswith("data:image/") + ): + return {"done": True, "issues": [], "retry_hint": "", + "reason": "unavailable", "text": "DONE"} + bases = local_bases() or list(DEFAULT_LOCAL_BASES) + if not bases: + return {"done": True, "issues": [], "retry_hint": "", + "reason": "unavailable", "text": "DONE"} + base = bases[0] + mdl = await _resolve_model(base, str(model or local_model()), local_model()) + messages = [{ + "role": "user", + "content": [ + {"type": "text", "text": str(prompt or "")}, + {"type": "image_url", "image_url": {"url": before}}, + {"type": "image_url", "image_url": {"url": after}}, + ], + }] + text = await _post_chat(base, mdl, messages, None, timeout, max_tokens=140) + if not text: + return {"done": True, "issues": [], "retry_hint": "", + "reason": "unavailable", "text": "DONE"} + out = _parse_self_check(text) + out["model"] = mdl + out["source"] = "local" + return out + + +def _resolve_override_candidate(override: str, owner: Optional[str]) -> Optional[Dict[str, Any]]: + """Resolve an explicit agent_model override under server authority. + + Guarantees: + - Client-provided raw URLs never determine arbitrary outbound network destinations. + - Registered endpoint IDs resolve only to enabled endpoints visible to the owner. + - Shared endpoints (owner IS NULL) remain visible to authorized users. + - Disabled, foreign-owned, forged, or unconfigured destinations fail closed (return None). + - Admin privileges do not allow arbitrary unconfigured destinations. + """ + raw = str(override or "").strip() + if not raw: + return None + + if "::" in raw: + target, requested_model = raw.split("::", 1) + target = target.strip() + requested_model = requested_model.strip() + else: + target = "" + requested_model = raw + + # 1. Explicit local selection or matching a configured local base + if target == "local": + bases = local_bases() + if not bases: + return None + return { + "base": bases[0], + "model": requested_model or local_model(), + "headers": None, + "source": "override", + } + for lb in local_bases(): + if target and target.rstrip("/") == lb.rstrip("/"): + return { + "base": lb, + "model": requested_model or local_model(), + "headers": None, + "source": "override", + } + + # 2. Explicit hosted fallback selection + try: + from src.endpoint_resolver import resolve_endpoint + + for prefix in ("utility", "default"): + try: + url, model, headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model: + if target in ("default", "hosted"): + return { + "base": url, + "model": requested_model or model, + "headers": headers, + "source": "hosted", + } + if target and target.rstrip("/") == url.rstrip("/"): + if not requested_model or requested_model == model: + return { + "base": url, + "model": requested_model or model, + "headers": headers, + "source": "hosted", + } + except Exception: + pass + if target in ("default", "hosted"): + return None + + # 3. Bare model id without endpoint prefix + if not target: + if requested_model == "local": + bases = local_bases() + return {"base": bases[0], "model": local_model(), "headers": None, "source": "local"} if bases else None + if requested_model in ("default", "hosted"): + return _resolve_override_candidate("default::", owner) + + # Exact match with local model + if requested_model == local_model(): + bases = local_bases() + if bases: + return {"base": bases[0], "model": local_model(), "headers": None, "source": "local"} + + # Exact match with hosted default/utility + try: + from src.endpoint_resolver import resolve_endpoint + + for prefix in ("utility", "default"): + try: + url, model, headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model and model == requested_model: + return {"base": url, "model": model, "headers": headers, "source": "hosted"} + except Exception: + pass + + # Check registered database endpoints for this bare model + try: + from core.database import SessionLocal, ModelEndpoint + from src.auth_helpers import owner_filter, _auth_disabled + from src.endpoint_resolver import ( + resolve_endpoint_runtime, + build_chat_url, + build_headers, + _endpoint_enabled_models, + _endpoint_hidden_models, + ) + + db = SessionLocal() + try: + if not owner and not _auth_disabled(): + q = db.query(ModelEndpoint).filter( + ModelEndpoint.is_enabled == True, + ModelEndpoint.owner == None, + ) + else: + q = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), + ModelEndpoint, + owner, + ) + matching_eps = [] + for ep in q.all(): + enabled_models = _endpoint_enabled_models(ep) + hidden_models = _endpoint_hidden_models(ep) + if requested_model in enabled_models and requested_model not in hidden_models: + matching_eps.append(ep) + + # Avoid model ID collision across providers: only accept if unique + if len(matching_eps) == 1: + ep = matching_eps[0] + base, api_key = resolve_endpoint_runtime(ep, owner=owner) + return { + "base": build_chat_url(base), + "model": requested_model, + "headers": build_headers(api_key, base), + "source": "registered", + "endpoint_id": ep.id, + } + finally: + db.close() + except Exception: + pass + return None + + # 4. Target is an endpoint_id or a registered legacy endpoint URL + try: + from core.database import SessionLocal, ModelEndpoint + from src.auth_helpers import owner_filter, _auth_disabled + from src.endpoint_resolver import ( + resolve_endpoint_runtime, + build_chat_url, + build_headers, + _endpoint_enabled_models, + _endpoint_hidden_models, + _first_chat_model, + ) + + db = SessionLocal() + try: + if not owner and not _auth_disabled(): + q = db.query(ModelEndpoint).filter( + ModelEndpoint.is_enabled == True, + ModelEndpoint.owner == None, + ) + else: + q = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), + ModelEndpoint, + owner, + ) + # Lookup by ID + ep = q.filter(ModelEndpoint.id == target).first() + + # If not found by ID, match legacy registered base URL (strictly registered rows only) + if ep is None and ("://" in target or target.startswith("http")): + target_norm = target.rstrip("/") + for cand_ep in q.all(): + cand_base = (cand_ep.base_url or "").rstrip("/") + if ( + cand_base == target_norm + or cand_base + "/v1" == target_norm + or cand_base == target_norm + "/v1" + ): + ep = cand_ep + break + + if ep is None: + # Target is disabled, foreign-owned, nonexistent, or arbitrary unconfigured URL -> fail closed! + return None + + enabled_models = _endpoint_enabled_models(ep) + hidden_models = _endpoint_hidden_models(ep) + if requested_model: + if requested_model in hidden_models: + return None + if enabled_models and requested_model not in enabled_models: + # Model not served by this endpoint -> fail closed against forged model IDs + return None + model = requested_model + else: + model = _first_chat_model(enabled_models) + if not model: + return None + + base, api_key = resolve_endpoint_runtime(ep, owner=owner) + return { + "base": build_chat_url(base), + "model": model, + "headers": build_headers(api_key, base), + "source": "registered", + "endpoint_id": ep.id, + } + finally: + db.close() + except Exception: + return None + + +def _candidates(owner: Optional[str], override: Optional[str]) -> List[Dict[str, Any]]: + cands: List[Dict[str, Any]] = [] + for base in local_bases(): + cands.append({"base": base, "model": local_model(), "headers": None, "source": "local"}) + try: + from src.endpoint_resolver import resolve_endpoint + + for prefix in ("utility", "default"): + try: + url, model, headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model: + cands.append({"base": url, "model": model, "headers": headers, "source": "hosted"}) + break + except Exception: + pass + if override: + cand = _resolve_override_candidate(override, owner) + if cand is not None: + cands.insert(0, cand) + return cands + # Unauthorized, invalid, foreign, disabled, or forged override: + # Fail closed! Never silently dispatch sensitive user images/state to an unexpected fallback. + return [] + return cands + + +async def plan_with_editor_agent( + messages: List[dict], + owner: Optional[str] = None, + override: Optional[str] = None, + timeout: float = 90.0, + response_format: Optional[dict] = None, +) -> Optional[Tuple[str, str, str]]: + """Return (raw_plan_text, model_used, source) or None. + + Tries the local model WITH the screenshot, then the same model text-only + with a VL caption of the screenshot, then the hosted chat model. + + ``response_format`` (a JSON-schema structured-output constraint) is sent + ONLY to the local editor-agent bases, so a hosted fallback model that does + not support the field never rejects the request. + """ + has_img = _has_image(messages) + text_messages = _strip_images(messages) if has_img else messages + image = _first_image(messages) if has_img else None + caption: Optional[str] = None + _local = {str(b).rstrip("/") for b in local_bases()} + for cand in _candidates(owner, override): + base, model = cand["base"], cand["model"] + headers = cand.get("headers") + is_local = cand["source"] == "local" or str(base).rstrip("/") in _local + rf = response_format if (response_format and is_local) else None + extra = {"response_format": rf} if rf else {} + if has_img: + raw = await _post_chat(base, model, messages, headers, timeout, **extra) + if raw: + return raw, model, cand["source"] + if caption is None and image: + caption = await caption_image(image, owner) + if caption: + raw = await _post_chat( + base, model, _with_caption(text_messages, caption), headers, timeout, **extra, + ) + if raw: + return raw, model, cand["source"] + "+caption" + else: + raw = await _post_chat( + base, model, text_messages, headers, timeout, **extra + ) + if raw: + return raw, model, cand["source"] + return None + + +async def list_editor_agent_models(owner: Optional[str] = None) -> List[Dict[str, Any]]: + """Models available to the editor agent, for the model picker. + + Lists the local Qwen3.5-9B (one entry per reachable base) plus the hosted + default/utility chat model that the planner falls back to. + """ + models: List[Dict[str, Any]] = [] + seen: set = set() + for base in local_bases(): + model = local_model() + key = ("local", base, model) + if key in seen: + continue + seen.add(key) + models.append({ + "id": model, + "label": f"{model} · local", + "source": "local", + "vision": True, + "base_url": base, + "endpoint_id": "local", + }) + try: + from src.endpoint_resolver import resolve_endpoint + from src.settings import get_user_setting, load_settings + + for prefix in ("utility", "default"): + try: + url, model, _headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model: + key = ("hosted", model) + if key not in seen: + seen.add(key) + ep_id = None + try: + settings = load_settings() + ep_id = (get_user_setting(f"{prefix}_endpoint_id", owner or "", settings.get(f"{prefix}_endpoint_id", "")) or "").strip() or None + except Exception: + pass + models.append({ + "id": model, + "label": model, + "source": "hosted", + "vision": None, + "base_url": url, + "endpoint_id": ep_id or prefix, + }) + break + except Exception: + pass + return models + + +# --------------------------------------------------------------------------- +# Reachability probe (best effort, non-blocking). +# --------------------------------------------------------------------------- + +_PROBE: Dict[str, bool] = {} + + +def probe_models(base: str, timeout: float = 3.0) -> bool: + try: + with httpx.Client(timeout=timeout) as client: + resp = client.get(str(base).rstrip("/") + "/models") + return resp.status_code == 200 + except Exception: + return False + + +def _probe_worker() -> None: + for base in local_bases(): + ok = probe_models(base) + _PROBE[base] = ok + logger.info("editor agent local base %s reachable=%s", base, ok) + cap = caption_base() + if cap: + _PROBE[cap] = probe_models(cap) + logger.info("editor agent caption base %s reachable=%s", cap, _PROBE[cap]) + + +def startup_probe() -> None: + """Kick off a background reachability probe (never blocks startup).""" + try: + threading.Thread(target=_probe_worker, name="editor-agent-probe", daemon=True).start() + except Exception: + logger.debug("editor agent probe thread failed", exc_info=True) + + +def probe_status() -> Dict[str, bool]: + return dict(_PROBE) diff --git a/src/editor_style_guide.py b/src/editor_style_guide.py new file mode 100644 index 000000000..1355c2608 --- /dev/null +++ b/src/editor_style_guide.py @@ -0,0 +1,1108 @@ +"""Single source of truth for photo-editor intent routing. + +Both the server-side planner (``routes/editor_agent_routes.py``) and the +browser fallback planner (``static/js/editor/nl-edit.js``) read this guide so +their behaviour cannot drift. It is served verbatim to the client at +``GET /api/editor/style-guide``. + +Calibrated strengths live here; this file is the only place to tune routing. +It is intentionally dependency-free so the route module can import it cheaply. +""" +from __future__ import annotations + +import json +import re +from collections import Counter +from typing import Any, Dict, List, Optional + +STYLE_GUIDE_VERSION = 8 + +# New editor ops (implemented client-side, static) with canonical params. The +# server op catalog must include these or a valid plan would be filtered out. +NEW_OPS: List[Dict[str, Any]] = [ + { + "op": "photo.autoTone", + "params": {}, + "help": "Camera-Raw-style auto tone/colour (per-channel levels, auto white " + "balance, exposure, contrast). The single preferred global lift.", + }, + { + "op": "retouch.skin", + "params": {"smooth": 8, "opacity": 0.35, "blemish": True, + "radius": 12, "threshold": 16}, + "help": "Gentle frequency-separation skin smoothing that keeps texture, over " + "the selection intersected with skin tones. Keep it subtle: smooth " + "<=14, opacity <=0.45; it never brightens the face.", + }, + { + "op": "retouch.blemish", + "params": {"radius": 4, "threshold": 16}, + "help": "Remove small blemishes inside the selection or skin region.", + }, + { + "op": "retouch.teeth", + "params": {"desat": 0.6, "bright": 16}, + "help": "Whiten teeth within the current selection (desaturate yellow, " + "lift brightness with a small feather).", + }, + { + "op": "selection.face", + "params": {"mode": "replace"}, + "help": "Select a face (SAM text 'face', upper-centre ellipse fallback).", + }, + { + "op": "selection.background", + "params": {"mode": "replace", "subject": "person", "fuzziness": 30}, + "help": "Select the background (SAM subject inverted, corner-colour " + "fallback). Subject-aware; prefer over a plain global blur.", + }, + { + "op": "fill.flood", + "params": {"color": "#ff0000", "x": 0, "y": 0, "tolerance": 32}, + "help": "Paint-bucket flood-fill a pixel layer from a point " + "{x,y,color,tolerance}. Use for 'fill ... with ' when there " + "is no active selection.", + }, + { + "op": "selection.fill", + "params": {"color": "#ff0000"}, + "help": "Fill the current selection with a colour on the active pixel layer " + "{color}. Use for 'fill the selection with '.", + }, + { + "op": "text.edit", + "params": {"content": "New text", "size": 96, "color": "#ff0000"}, + "help": "Edit an EXISTING text layer {id|index|name, content, font, size, " + "color, align, tracking, lineHeight}. Use for 'make the text " + "bigger/smaller' and 'make the title ' — never text.add.", + }, + { + "op": "layers.align", + "params": {"mode": "center"}, + "help": "Align/distribute the selected layers to the canvas {mode:left|center|" + "right|top|middle|bottom|distribute-h|distribute-v}.", + }, + { + "op": "layer.rotate", + "params": {"degrees": 90}, + "help": "Rotate the ACTIVE LAYER by 90/180/270 {degrees, id|index|name}. Use " + "unless the prompt names the whole image/canvas/photo.", + }, + { + "op": "layer.flip", + "params": {"axis": "h"}, + "help": "Flip the ACTIVE LAYER in place {axis:h|v, id|index|name}.", + }, + { + "op": "document.rotate", + "params": {"degrees": 90}, + "help": "Rotate the WHOLE document (all layers + canvas) by 90/180/270 " + "{degrees} — for 'rotate the whole image/canvas/photo'.", + }, + { + "op": "document.flip", + "params": {"axis": "h"}, + "help": "Flip the WHOLE document {axis:h|v} — for 'flip the canvas/whole " + "image'.", + }, + { + "op": "selection.copy", + "params": {}, + "help": "Copy (or cut, {cut:true}) the selection pixels to the internal " + "clipboard.", + }, + { + "op": "selection.paste", + "params": {}, + "help": "Paste the internal clipboard as a new layer {name}.", + }, + { + "op": "layers.flatten", + "params": {}, + "help": "Flatten all layers into a single layer.", + }, + { + "op": "layers.mergeDown", + "params": {}, + "help": "Merge the active layer down into the layer below.", + }, +] + +NEW_OP_NAMES: List[str] = [op["op"] for op in NEW_OPS] + + +def _adj(kind: str, params: Dict[str, Any]) -> Dict[str, Any]: + return {"op": "adjustments.apply", "params": {"type": kind, "params": params}} + + +def _filter(fid: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + return {"op": "filters.apply", "params": {"id": fid, **(params or {})}} + + +# Ordered intent routes. The first route whose keyword appears wins. +INTENT_ROUTES: List[Dict[str, Any]] = [ + { + "id": "teeth", + "keywords": ["teeth", "whiten my smile", "whiter smile", "smile looks"], + "steps": [ + {"op": "selection.face", "params": {"mode": "replace"}}, + {"op": "retouch.teeth", "params": {"desat": 0.6, "bright": 16}}, + ], + "reply": "Selecting the face and whitening teeth.", + }, + { + "id": "portrait_retouch", + "keywords": ["skin", "complexion", "blemish", "acne", "pores", "retouch", + "look better", "look my best", "make me look", "beautify", + "flawless", "smooth my face", "clear up", "clean up my face", + "portrait", "selfie"], + # Do NOT lead with a whole-image photo.autoTone: lightening the whole frame + # washes out the face (the client suppresses autoTone for portrait intents + # unless the histogram shows real under/over-exposure). Go straight to a + # gentle skin retouch; keep it subtle and never brighten the face. + "min_steps": 3, + "steps": [ + {"op": "selection.face", "params": {"mode": "replace"}}, + {"op": "retouch.skin", "params": {"smooth": 8, "opacity": 0.35, + "blemish": True, "radius": 12, + "threshold": 16}}, + {"op": "selection.deselect", "params": {}}, + ], + "reply": "Selecting the face and gently retouching the skin.", + }, + { + "id": "restore", + "keywords": ["restore", "restoration", "old photo", "old photograph", + "old picture", "faded photo", "faded picture", "faded", + "recover this photo", "bring back this photo", + "photo restoration"], + # Never a large exposure lift: recover contrast + colour from a faded + # baseline, then a slight sharpen. + "min_steps": 4, + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("brightness-contrast", {"brightness": 0, "contrast": 18}), + _adj("white-balance", {"temperature": 0, "tint": 5}), + _adj("vibrance", {"vibrance": 18}), + _filter("sharpen"), + ], + "reply": "Restoring contrast and colour, then a light sharpen.", + }, + { + "id": "background", + "keywords": ["background"], + "steps": [ + {"op": "selection.background", + "params": {"mode": "replace", "subject": "person", "fuzziness": 30}}, + {"op": "filters.applyMasked", "params": {"id": "blur-gaussian"}}, + ], + "reply": "Selecting the background and softening it.", + }, + { + "id": "lighting", + # Low-key lighting: one auto-tone pass (the preferred global lift) plus + # at most one shadow-lifting, highlight-protecting adjustment. Both are + # sanctioned together; never a stack of brightness/exposure lifts. + "keywords": ["lighting", "exposure", "too dark", "underexposed", + "underexpose", "backlit", "back-lit", "brighten", "brighter", + "lift the shadows", "dark photo", "dark image"], + "min_steps": 2, + "steps": [ + {"op": "photo.autoTone", "params": {}}, + {"op": "adjustments.apply", + "params": {"type": "shadows-highlights", + "params": {"shadows": 30, "highlights": -15}}}, + ], + "reply": "Auto tone (levels and white balance), then one shadow lift " + "that protects the highlights.", + }, + { + "id": "cool", + "keywords": ["moodier", "moody", "cooler", "cool it", "cinematic", "teal", + "blue tone", "cold", "night mood", "colder"], + "steps": [_adj("white-balance", {"temperature": -25})], + "reply": "Cooling the white balance (negative temperature).", + }, + { + "id": "warm", + "keywords": ["warmer", "warm it", "golden", "summer", "sunset", + "golden hour", "warmth"], + "steps": [_adj("white-balance", {"temperature": 25})], + "reply": "Warming the white balance (positive temperature).", + }, + { + "id": "black_white", + "keywords": ["black and white", "black & white", "grayscale", "greyscale", + "monochrome", "b&w"], + "steps": [_adj("black-white", {})], + "reply": "Converting to black and white.", + }, + { + "id": "polish", + # Generic "make it look good" intents: a decisive multi-step grade with + # visible strength (a single timid adjustment scores as imperceptible). + "keywords": ["pretty", "beautiful", "make it look good", "look good", + "professional", "professionally", "editorial", "magazine", + "glam", "glamour", "pop", "punchy", "striking", "vibrant", + "film", "film look", "cinematic look", "instagram", + "social media", "elevate", "polish"], + "min_steps": 3, + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("vibrance", {"vibrance": 25}), + _filter("clarity", {"amount": 22, "radius": 30}), + _filter("vignette", {"amount": 22, "feather": 0.6}), + ], + "reply": "Balancing tone, adding vibrance and local contrast, with a light vignette.", + }, + { + "id": "vivid", + "keywords": ["vivid", "dramatic", "saturate", "saturation", + "colourful", "colorful"], + "steps": [_adj("vibrance", {"vibrance": 35})], + "reply": "Boosting vibrance.", + }, + { + "id": "sharpen", + "keywords": ["sharpen", "crisper", "crisp", "more detail", "sharper"], + "steps": [_filter("sharpen")], + "reply": "Sharpening.", + }, +] + +# A prompt that only asks for a vague "improvement" with no concrete defect and +# no matching intent cannot be planned deterministically -> ask the user. +_AMBIGUOUS_PATTERNS = [ + "clean this up", "clean it up", "fix it", "fix this", "make it better", + "make this better", "improve", "touch up", "touch it up", "do something", + "enhance", "make it nice", "polish", +] +_AMBIGUOUS_OPTIONS = [ + {"label": "Auto tone & color", + "description": "photo.autoTone: balanced levels, white balance, contrast."}, + {"label": "Brighten (lift shadows)", + "description": "shadows-highlights: brighten without blowing highlights."}, + {"label": "Cooler / cinematic", + "description": "white-balance negative: cooler, moodier look."}, +] +_DEFECT_HINTS = [ + "skin", "teeth", "background", "light", "dark", "colour", "color", "warm", + "cool", "blur", "sharpen", "contrast", "bright", "shadow", "face", + "blemish", "noise", "grain", "crop", "vibrance", "saturat", "black and white", + "grayscale", "portrait", "moody", "exposure", "tone", +] + +# Low-key safety: at most ONE global lift; prefer shadows/highlights and keep +# highlights protected. Global-lift adjustment types are counted together. +GLOBAL_LIFT_TYPES = ("autoTone", "brightness-contrast", "exposure", "levels", + "shadows-highlights") +_SAFE_LOWKEY_LIFT = { + "op": "adjustments.apply", + "params": {"type": "shadows-highlights", + "params": {"shadows": 30, "highlights": -15}}, +} + + +def match(prompt: str) -> Optional[Dict[str, Any]]: + """Return the first matching intent route for a prompt, or None.""" + text = str(prompt or "").lower() + for route in INTENT_ROUTES: + if any(keyword in text for keyword in route["keywords"]): + return route + return None + + +def is_ambiguous(prompt: str) -> bool: + text = str(prompt or "").lower() + if match(text): + return False + if any(hint in text for hint in _DEFECT_HINTS): + return False + return any(pattern in text for pattern in _AMBIGUOUS_PATTERNS) + + +def ask_user_plan() -> Dict[str, Any]: + question = "What would you like me to do with this photo?" + return { + "reply": question, + "planner": "fallback", + "steps": [], + "ask_user": { + "question": question, + "options": [dict(o) for o in _AMBIGUOUS_OPTIONS], + "multi": False, + }, + } + + +def has_concrete_intent(prompt: str) -> bool: + """True when a prompt maps to a concrete edit (a recipe, look, recolour, + structural op) or names a concrete defect. Used to decide whether a plan made + only of generic whole-image tone ops should be trusted or replaced by an + ask_user prompt. + """ + text = str(prompt or "").lower() + if match(text) or look_plan(text) or recolor_plan(text) or structural_plan(text): + return True + return any(hint in text for hint in _DEFECT_HINTS) + + +def fallback_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan from the guide, or None when nothing matches.""" + route = match(prompt) + if route: + return { + "reply": route["reply"], + "planner": "fallback", + "steps": enforce_guards([dict(step) for step in route["steps"]]), + } + if is_ambiguous(prompt): + return ask_user_plan() + return None + + +# ── Structural op routing ────────────────────────────────────────────── +# Deterministic plans for structural editor ops the sampling model reliably +# substitutes with a generic adjustments/transform step. Mirrors the browser +# fallback (static/js/editor/nl-edit.js localPlan) so the server and client +# agree; the server path must emit these or a wrong model plan suppresses the +# browser fallback. +_NAMED_COLORS = { + "red": "#ff0000", "blue": "#0000ff", "green": "#00a000", "yellow": "#ffff00", + "orange": "#ff8000", "purple": "#8000ff", "pink": "#ff69b4", "black": "#000000", + "white": "#ffffff", "grey": "#808080", "gray": "#808080", "cyan": "#00ffff", + "magenta": "#ff00ff", "brown": "#8b4513", "teal": "#008080", +} +_COLOR_TOKEN = (r"(red|blue|green|yellow|orange|purple|pink|black|white|grey|" + r"gray|cyan|magenta|brown|teal|#[0-9a-fA-F]{3,6})") + + +def color_for_name(name: str) -> Optional[str]: + n = str(name or "").strip().lower() + if re.match(r"^#[0-9a-f]{3,6}$", n): + return n + return _NAMED_COLORS.get(n) + + +# ── Named reference looks + targeted recolour ────────────────────────── +# Server-only deterministic plans (reached via routes._plan_special -> +# structural_plan). Deliberately NOT added to INTENT_ROUTES: the browser +# client consumes `intents` and cannot run an ai_edit_image step, so strong +# looks must stay on the server path. +_LOOK_KEEP_CLAUSE = " Keep the main subject and composition exactly the same." + +LOOKS = [ + { + "id": "wes_anderson", + "keywords": ["wes anderson"], + "mode": "recipe", + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("white-balance", {"temperature": 10, "tint": 6}), + _adj("vibrance", {"vibrance": 18}), + _adj("brightness-contrast", {"brightness": 6, "contrast": -10}), + ], + "reply": "Gave it a Wes Anderson look: pastel palette, warm highlights and soft contrast.", + }, + { + "id": "film_noir", + "keywords": ["film noir"], + "mode": "recipe", + "steps": [ + _adj("black-white", {}), + _adj("brightness-contrast", {"contrast": 38}), + _adj("shadows-highlights", {"shadows": -22, "highlights": 18}), + _filter("vignette", {"amount": 45, "feather": 0.5}), + ], + "reply": "Gave it a film-noir look: high-contrast black and white with deep shadows.", + }, + { + "id": "vintage_kodachrome", + "keywords": ["kodachrome", "vintage film"], + "mode": "recipe", + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("white-balance", {"temperature": 12, "tint": 8}), + _adj("vibrance", {"vibrance": 22}), + _adj("brightness-contrast", {"contrast": 14}), + _filter("vignette", {"amount": 12, "feather": 0.7}), + ], + "reply": "Gave it a vintage Kodachrome look: warm saturated colour and lifted contrast.", + }, + { + "id": "cyberpunk", + "keywords": ["cyberpunk"], + "mode": "instruction", + "prompt": "Restyle the photo as a cyberpunk scene: vivid neon magenta and cyan " + "lighting, wet reflective surfaces, deep shadows and a futuristic " + "atmosphere." + _LOOK_KEEP_CLAUSE, + "reply": "Restyling it as a cyberpunk scene with the image model.", + }, + { + "id": "studio_ghibli", + "keywords": ["ghibli", "studio ghibli"], + "mode": "instruction", + "prompt": "Restyle the photo as a soft hand-painted anime scene in a " + "Studio-Ghibli-inspired pastel style: luminous pastel palette, " + "gentle painterly shading, clean edges and warm sunlight." + + _LOOK_KEEP_CLAUSE, + "reply": "Restyling it as a Studio-Ghibli-inspired pastel scene with the image model.", + }, +] + + +def look_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan for a named reference look, or None. + + Recipe looks (Wes Anderson, film noir, vintage Kodachrome) are pure + adjustment/filter stacks; strong looks (cyberpunk, Studio Ghibli) use + ai_edit_image instruction mode. + """ + p = str(prompt or "").lower() + for look in LOOKS: + if any(k in p for k in look["keywords"]): + if look["mode"] == "instruction": + steps = [{"op": "ai_edit_image", + "params": {"op": "generative", "mode": "instruction", + "prompt": look["prompt"], "steps": 8}}] + else: + steps = [dict(s) for s in look["steps"]] + return {"reply": look["reply"], "planner": "deterministic", "steps": steps} + return None + + +_RECOLOR_ACTION_RE = re.compile( + r"\brecolou?r\b" + r"|\bmake\s+only\b" + r"|\bmake\s+the\b[^.]{0,30}?\b" + _COLOR_TOKEN + r"\b" + r"|\b(?:change|turn|convert|paint|colou?r)\b[^.]{0,40}?\b" + _COLOR_TOKEN + r"\b", + re.I, +) +_RECOLOR_EXCLUDE_RE = re.compile( + r"\b(?:background|backdrop|blur|black\s+and\s+white|grayscale|greyscale|" + r"monochrome|b&w|text|title|font|caption|label|logo|watermark|drop\s+shadow|" + r"generative)\b", + re.I, +) + + +_RECOLOR_TRAILING_ADJECTIVES = frozenset({ + "bright", "brighter", "vivid", "vibrant", "dark", "darker", "light", + "lighter", "deep", "pale", "more", "less", "very", "really", + "slightly", "totally", "completely", +}) + + +def _hex_rgb01(hex_color: str): + h = str(hex_color or "").lstrip("#") + if len(h) == 3: + h = "".join(ch * 2 for ch in h) + try: + return tuple(int(h[i:i + 2], 16) / 255.0 for i in (0, 2, 4)) + except Exception: + return None + + +def _hue_degrees(hex_color: str) -> Optional[float]: + rgb = _hex_rgb01(hex_color) + if rgb is None: + return None + r, g, b = rgb + mx, mn = max(rgb), min(rgb) + if mx == mn: + return 0.0 + d = mx - mn + if mx == r: + h = ((g - b) / d) % 6 + elif mx == g: + h = (b - r) / d + 2 + else: + h = (r - g) / d + 4 + return h * 60.0 + + +def _hue_delta(src_hex: str, tgt_hex: str) -> int: + sh = _hue_degrees(src_hex) + th = _hue_degrees(tgt_hex) + if sh is None or th is None: + return 0 + return int(round((th - sh + 180.0) % 360.0 - 180.0)) + + +def _is_chromatic(hex_color: str) -> bool: + """True when a pure hue rotation can plausibly reach this colour. + + White / black / grey targets have (near) zero saturation and cannot be + reached by rotating hue, so they go through the image model instead. + """ + rgb = _hex_rgb01(hex_color) + if rgb is None: + return False + mx, mn = max(rgb), min(rgb) + return (mx - mn) >= 0.15 and mx >= 0.12 + + +def _recolor_object_phrase(prompt: str, target_word: str) -> str: + """Best-effort object description for the mask ("red car" in 'the red car').""" + m = re.search( + r"\b(?:make|recolou?r|change|turn|convert|paint|colou?r)\b\s+(?:only\s+)?(.+)", + prompt, re.I, + ) + phrase = (m.group(1) if m else prompt).strip() + if target_word: + phrase = re.sub(r"\b" + re.escape(target_word) + r"\b", " ", phrase, + count=1, flags=re.I) + words = " ".join(phrase.split()).strip(" .,!?").split() + if len(words) > 1 and words[0].casefold() in {"to", "into", "the", "a", "an"}: + words.pop(0) + if len(words) > 1 and words[-1].casefold() in {"to", "into"}: + words.pop() + while len(words) > 1 and words[-1].lower() in _RECOLOR_TRAILING_ADJECTIVES: + words.pop() + phrase = " ".join(words) + return phrase or "the subject" + + +def recolor_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan for a targeted recolour, or None. + + 'make only the red car blue' -> select the OBJECT by description + (``selection.samText`` "red car"; falling back to a colour-range selection + via ``fallbackColor`` when the mask is empty) and then shift the MASKED + HUE toward the target with ``adjustments.applyMasked`` (preserving shading, + highlights and reflections). Never a flat ``selection.fill`` and never a + whole-image ``adjustments.apply`` tint. When the target cannot be reached by + a hue rotation (white/black/grey) or no source colour is known, the masked + region is recoloured with ``ai_edit_image`` instruction mode instead. + """ + p = str(prompt or "") + if not _RECOLOR_ACTION_RE.search(p): + return None + if _RECOLOR_EXCLUDE_RE.search(p): + return None + colors = [m.group(1).lower() for m in + re.finditer(r"\b" + _COLOR_TOKEN + r"\b", p)] + colors = [c for c in colors if color_for_name(c)] + if not colors: + return None + target = colors[-1] + tgt_hex = color_for_name(target) + source = colors[0] if len(colors) >= 2 and colors[0] != target else None + src_hex = color_for_name(source) if source else None + phrase = _recolor_object_phrase(p, target) + select = { + "op": "selection.samText", + "params": {"text": phrase}, + } + if src_hex: + select["params"]["fallbackColor"] = src_hex + select["params"]["fallbackFuzziness"] = 40 + if source and src_hex and _is_chromatic(tgt_hex): + delta = _hue_delta(src_hex, tgt_hex) + src_hue = int(round(_hue_degrees(src_hex) or 0)) + return { + "reply": "Selected the %s and shifted its hue toward %s " + "(shading kept)." % (phrase, target), + "planner": "deterministic", + "steps": [ + select, + {"op": "adjustments.applyMasked", + "params": {"hue": delta, "saturation": 0, + "mode": "hue-saturation", + "sourceHue": src_hue, + "source": src_hex, "target": tgt_hex}}, + ], + } + instruction = ("Recolour the %s to %s, preserving its shading, highlights " + "and reflections." % (phrase, target)) + return { + "reply": "Recolouring the %s with the image model." % phrase, + "planner": "deterministic", + "steps": [ + select, + {"op": "ai_edit_image", + "params": {"op": "inpaint", "mode": "masked", + "prompt": instruction, "steps": 8}}, + ], + } + + +def structural_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan for structural editor ops, or None. + + Covers fill/flood, text.edit, colour-overlay layer style, layer align, + layer/document rotate+flip, selection copy/paste, flatten and merge-down. + Also dispatches named reference looks and targeted recolours. + Returns ``{"reply", "planner": "deterministic", "steps": [...]}``. + """ + look = look_plan(prompt) + if look is not None: + return look + recolor = recolor_plan(prompt) + if recolor is not None: + return recolor + p = str(prompt or "").lower() + steps: List[Dict[str, Any]] = [] + labels: List[str] = [] + + fm = (re.search(r"\bfill\b[^.]*?\bwith\s+" + _COLOR_TOKEN + r"\b", p) + or re.search(r"\bfill\b[^.]*?\b" + _COLOR_TOKEN + r"\b", p)) + if fm: + color = color_for_name(fm.group(1)) + if color: + if re.search(r"\bselection\b|\bselected\b", p): + steps.append({"op": "selection.fill", "params": {"color": color}}) + labels.append("filled the selection") + else: + steps.append({"op": "fill.flood", + "params": {"color": color, "x": 0, "y": 0}}) + labels.append("flood-filled the layer") + + if re.search(r"\b(text|title|caption|font|label)\b", p) and not re.search( + r"\b(add|insert|create|write|put|place)\b", p): + tparams: Dict[str, Any] = {} + if re.search(r"\b(bigger|larger|increase)\b", p): + tparams["size"] = 96 + elif re.search(r"\b(smaller|reduce|decrease)\b", p): + tparams["size"] = 28 + cm = re.search(r"\b" + _COLOR_TOKEN + r"\b", p) + if cm: + c = color_for_name(cm.group(1)) + if c: + tparams["color"] = c + if tparams: + steps.append({"op": "text.edit", "params": tparams}) + labels.append("updated the text layer") + + if re.search(r"\bcolo?ur?\s*overlay\b", p): + steps.append({"op": "layers.addStyle", + "params": {"type": "color-overlay", "params": {}}}) + labels.append("added a colour-overlay layer style") + + am = re.search( + r"\balign\b[^.]*?\b(centre|center|left|right|top|middle|bottom)\b", p) + if am: + mode = "center" if am.group(1) in ("centre", "center") else am.group(1) + steps.append({"op": "layers.align", "params": {"mode": mode}}) + labels.append("aligned to the %s" % mode) + + whole = bool(re.search( + r"\b(?:whole|entire|full)\s+(?:image|canvas|photo|picture|document)\b", p)) \ + or bool(re.search(r"\b(?:canvas|document)\b", p) + and re.search(r"\b(?:rotate|flip|mirror|turn)\b", p)) + if re.search(r"\b(?:flip|mirror)\b", p): + axis = "v" if re.search(r"\b(?:vertical(?:ly)?|v)\b", p) else "h" + if whole: + steps.append({"op": "document.flip", "params": {"axis": axis}}) + labels.append("flipped the whole image") + else: + steps.append({"op": "layer.flip", "params": {"axis": axis}}) + labels.append("flipped the layer") + # Rotation needs an explicit angle/quarter-turn or the verb "rotate"; a bare + # "turn ... into