diff --git a/app.py b/app.py index b0e6f9d55..ae859a579 100644 --- a/app.py +++ b/app.py @@ -178,6 +178,14 @@ app.add_middleware(SecurityHeadersMiddleware) import asyncio as _asyncio from starlette.middleware.base import BaseHTTPMiddleware as _BaseHTTPMiddleware from starlette.responses import JSONResponse as _JSONResponse +import anyio as _anyio +from starlette.requests import ClientDisconnect as _ClientDisconnect + +# SSE endpoints (e.g. /api/editor/bridge/events) end/abort mid-stream when the +# browser disconnects; Starlette's BaseHTTPMiddleware surfaces that as +# anyio.EndOfStream (and sometimes ClientDisconnect). Those are client-side +# events, not server errors, so we map them to a 499 instead of a 500 traceback. +_STREAM_CLIENT_EXC = (_anyio.EndOfStream, _ClientDisconnect) REQUEST_HARD_TIMEOUT = float(os.getenv("REQUEST_HARD_TIMEOUT", "45")) _TIMEOUT_EXEMPT_PREFIXES = ( @@ -190,6 +198,7 @@ _TIMEOUT_EXEMPT_PREFIXES = ( "/api/cookbook/setup", # remote pacman/apt installs "/api/upload", # large files "/api/image", # diffusion proxies (inpaint/harmonize/upscale/etc.) — own 120s httpx timeout + "/api/editor", # editor agent/bridge: generative steps can take 30-110s (SAM + local diffusion) "/api/memory/audit", # retains own 120s LLM inactivity timeout ) @@ -225,15 +234,43 @@ class _InteractiveActivityMiddleware(_BaseHTTPMiddleware): return await call_next(request) +# Heavy image ops that can run 30-150 s; a deploy restart must not cut these off. +_HEAVY_IMAGE_OP_PREFIXES = ( + "/api/image/inpaint", + "/api/image/replace-background", + "/api/image/outpaint", + "/api/image/extend", + "/api/image/remove-bg", + "/api/image/mask", + "/api/image/upscale-local", + "/api/image/denoise", + "/api/image/enhance-face", + "/api/gallery/ai-upscale", + "/api/gallery/style-transfer", + "/api/editor/agent", +) + + class _SlowRequestLogMiddleware(_BaseHTTPMiddleware): async def dispatch(self, request, call_next): + from src import activity + path = request.url.path or "" + heavy = any(path.startswith(p) for p in _HEAVY_IMAGE_OP_PREFIXES) + if heavy: + activity.begin_image_op(path) start = time.perf_counter() status = 500 try: response = await call_next(request) status = getattr(response, "status_code", 0) or 0 return response + except _STREAM_CLIENT_EXC: + # Client disconnected / SSE stream ended — not a server error. + status = 499 + return _JSONResponse({"detail": "client disconnected"}, status_code=499) finally: + if heavy: + activity.end_image_op(path) elapsed = time.perf_counter() - start try: threshold = float(os.getenv("ODYSSEUS_SLOW_REQUEST_LOG_SECONDS", "0.75") or "0.75") @@ -822,6 +859,10 @@ app.include_router(setup_gallery_routes()) from routes.editor_draft_routes import setup_editor_draft_routes app.include_router(setup_editor_draft_routes()) +# Natural-language editor planning (planner wired to the configured chat model) +from routes.editor_agent_routes import setup_editor_agent_routes +app.include_router(setup_editor_agent_routes()) + # Scheduled tasks + event bus from src.task_scheduler import TaskScheduler task_scheduler = TaskScheduler(session_manager) @@ -1000,6 +1041,16 @@ async def get_version(): "source_commit": APP_SOURCE_COMMIT, } + +@app.get("/api/busy") +async def get_busy(): + """Report in-flight heavy work so a deploy can wait instead of dropping it.""" + from src import activity + return { + "busy_image_ops": activity.heavy_image_ops_active(), + **activity.activity(), + } + @app.get("/api/health") async def health_check() -> Dict[str, str]: return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()} @@ -1438,4 +1489,35 @@ if __name__ == "__main__": bind_host = os.getenv("APP_BIND", "127.0.0.1") bind_port = int(os.getenv("APP_PORT", "7011")) - uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") + # Give in-flight requests time to finish on SIGTERM instead of being cut + # off mid-flight. Diffusion edits run 60-120 s; outpaint/extend can run up + # to ~330 s (provider cap 900 s), so default the graceful window to 900 s. + # systemd's TimeoutStopSec must exceed this (see the graceful-stop drop-in). + # + # We hook uvicorn's signal handler so a SIGTERM also flips src.shutdown: + # SSE / streaming generators check it and end promptly, so an open browser + # stream cannot stall a restart for the whole window. The long window stays + # a safety net for genuine in-flight image ops. + from src import shutdown as _shutdown + + config = uvicorn.Config( + app, + host=bind_host, + port=bind_port, + log_level="info", + timeout_graceful_shutdown=int(os.getenv("ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT", "900") or 900), + ) + server = uvicorn.Server(config) + + # uvicorn's capture_signals() wires SIGINT/SIGTERM to self.handle_exit, so + # wrapping that method lets us set the shutdown flag before the normal exit + # path runs (this uvicorn version has no install_signal_handlers kwarg). + _uvicorn_handle_exit = server.handle_exit + + def _handle_shutdown_signal(signum, frame): + _shutdown.request_shutdown() + _uvicorn_handle_exit(signum, frame) + + server.handle_exit = _handle_shutdown_signal + + server.run() diff --git a/docs/EDITOR_AGENT_API.md b/docs/EDITOR_AGENT_API.md new file mode 100644 index 000000000..c873d28d9 --- /dev/null +++ b/docs/EDITOR_AGENT_API.md @@ -0,0 +1,201 @@ +# Editor Agent API + +How an agent (LLM or script) drives the PhotoCraft editor. There are three layers: + +1. **Control API** — `window.geEditorControl`, a stable in-page command surface. +2. **NL planner** — `window.geEditorAI`, turns a prompt into control-API steps (LLM or basic mode). +3. **Bridge** — server-mediated SSE channel so a *server-side* agent can drive an **open** editor. + +All three are owner-scoped by the app session; nothing is exposed to unauthenticated users. + +--- + +## 1. Control API (`window.geEditorControl`) + +Installed when the editor opens. Version `EDITOR_CONTROL_VERSION`. + +```js +await window.geEditorControl.invoke(op, params) // { ok, op, result } | { ok:false, op, error } +await window.geEditorControl.run([{op, params}]) // sequential, stops on first failure +window.geEditorControl.list() // [{op, description}] +``` + +`invoke` is async. Every op that mutates the document pushes **one** undo entry via +`_saveState`, so a single op is one undo step. + +### Op catalog + +| Op | Params | Notes | +| --- | --- | --- | +| `document.info` | – | size, layers, mode | +| `document.importImage` | `{dataUrl,name,raster}` | load a raster image as a new layer | +| `layers.list` | – | top-most first | +| `layers.select` | `{id\|index\|name}` | index is top-most-first | +| `layers.addRaster` | – | new empty pixel layer | +| `layers.addAdjustment` | `{type,params}` | adjustment layer | +| `layers.duplicate` / `layers.delete` | – | on the active/selected layer(s) | +| `layers.setOpacity` | `{id?,opacity 0-1}` | | +| `layers.setBlend` | `{id?,blendMode\|mode}` | | +| `layers.group` / `layers.ungroup` | – | | +| `selection.selectAll` / `deselect` / `invert` | – | | +| `selection.rectangle` / `selection.ellipse` | `{x,y,w,h}` | document space | +| `selection.subject` | `{label}` | SAM text grounding (falls back to a centred rect) | +| `selection.face` | – | SAM "face" (falls back to an upper-centre ellipse) | +| `selection.sky` | – | SAM "sky" (falls back to a top-band colour range) | +| `selection.background` | `{fuzziness}` | border colour union | +| `selection.samText` | `{text}` | SAM text grounding only (throws without SAM) | +| `selection.colorRange` | `{color,fuzziness}` | | +| `selection.hasMask` | – | `{active, space}` | +| `adjustments.apply` | `{type,params}` | adjustment layer; camelCase types accepted | +| `adjustments.applyMasked` | `{brightness,contrast,saturation,warmth,invert}` | pixel pass, masked if a selection exists else whole layer | +| `filters.apply` | `{id}` | opens the slider dialog and applies | +| `filters.applyMasked` | `{id,radius,amount,invert}` | `id`: blur-gaussian, sharpen, unsharp-mask | +| `transform.start` / `transform.apply` / `transform.cancel` | – | | +| `text.add` | `{text,x,y,color,size,font,align}` | | +| `history.undo` / `history.redo` | – | | +| `history.mark` | – | returns current undo depth (for grouping) | +| `history.collapse` | `{mark,label}` | collapses entries above `mark` into one | +| `export.png` | – | `{dataUrl}` of the flattened document | +| `export.gallery` | – | saves a copy back to the gallery | +| `screenshot` | – | `{dataUrl,width,height}` of the composited canvas | + +Param conventions the planner must respect: + +- `adjustments.applyMasked`: `brightness`/`contrast`/`warmth` are **-100..100** (0 neutral); + `saturation` is a **multiplier** (1 neutral; values >3 are treated as percentages). +- `adjustments.apply` `type` aliases are normalised (`blackAndWhite` -> `black-white`). +- `color-balance` params are `{shadows:{r,g,b}, midtones:{r,g,b}, highlights:{r,g,b}}`; + a bare number is coerced to `{r:n,g:n,b:n}`. +- A masked op with **no** active selection degrades to a whole-layer edit (does not throw). + +--- + +## 2. NL planner (`window.geEditorAI`) + +```js +const res = await window.geEditorAI.execute('blur the background behind the person', { onStep }); +// exec: window.geEditorAI.execute('blur the background behind the person', { onStep }); +// res = { ok, prompt, reply, planner, basicMode, steps, trace, failed } +``` + +Flow (`static/js/editor/nl-edit.js`): + +1. Read document state + a **before** screenshot. +2. Ask the **server planner** (`POST /api/editor/plan`) with the prompt, state and + screenshot. The client passes its strong guidance via `state.planner_guidance`. + If the route is unavailable it falls back to `localPlan` (regex — labelled + **"basic mode"** in the UI). +3. `sanitizeSteps()`: drop no-op/export steps and, if a masked op has no preceding + `selection.*`, insert the region implied by the prompt (face/sky/background/subject). +4. `history.mark`, then run each step, screenshotting after each one. +5. **Self-check** (server plans only, max 3 rounds): send the after screenshot back in + `mode:"check"` with the original intent. If not satisfied, **revert** the previous + attempt (undo its history entries) and apply the model's **full corrected** plan. +6. `history.collapse` — the whole request becomes a single undoable step. + +UI: the Quick Edit box routes multi-clause / "and ..." prompts to this executor and +streams `Step i/n: ` into its status line; a `(basic mode)` suffix marks the offline planner. + +### Server route + +`POST /api/editor/plan` (HARNESS, `routes/editor_agent_routes.py`) + +``` +{ prompt, state, mode?: "plan"|"check", screenshot?: "data:image/..." } +-> { reply, planner, steps: [{op, params}], done? } +``` + +Owner-scoped, authenticated. Uses the configured default/utility model with vision. +If the model call is unavailable it returns a deterministic `planner:"fallback"` plan. + +--- + +## 3. Bridge (server agent drives an open editor) + +`static/js/editor/editor-bridge.js` subscribes the open editor to the server queue. +Enabled automatically when the editor opens (`window.__geEditorBridge`). + +- Subscribe: `GET /api/editor/bridge/events?session=` — SSE, `data: {id,op,params}` (keepalive every 15s). +- The page runs the op via the control API and replies: + `POST /api/editor/bridge/result {id,ok,result,error,screenshot,ms}`. +- Enqueue a command (server side): `POST /api/editor/bridge/command {session, op|steps, wait, timeout}`. + With `wait:true` the call blocks and returns the editor's result. + +Round-trip example (authenticated; `` = `window.__geEditorBridge.session`): + +```bash +curl -s -X POST http://127.0.0.1:7011/api/editor/bridge/command \ + -H 'Content-Type: application/json' -b "odysseus_session=$TOKEN" \ + -d '{"session":"default","op":"layers.addRaster","wait":true,"timeout":8000}' +# -> {"id":"...","ok":true,"result":{...},"screenshot":"data:image/png;base64,..."} +``` + +Session scoping is `owner:session`, so an agent can only drive editors owned by the +same user. The bridge is a **browser-only** flow: it requires an editor page to be open. + +--- + +## 4. SAM dependency + +`selection.subject` / `selection.face` / `selection.sky` / `selection.samText` use the +server SAM mask tool (`POST /api/image/mask`). When the SAM tools are **not installed** +the editor raises *"Object mask tools are not installed"* and the control op falls back +to a **coarse approximation** (centred rect / upper-centre ellipse / top-band colour range). +On a plain background the approximation is acceptable; on real photos it leaks, producing +visible rectangular/elliptical artifacts. Install the Cookbook SAM mask dependencies for +true subject/face/sky segmentation. + +`selection.colorRange` and `selection.background` never need SAM (pure colour matching), +but they are also coarse on busy scenes. + +--- + +## 5. Two ways for an agent to use this — pick one + +### Option A — built-in tool (in-process) + +Register an editor-control tool inside the app (same process as the model) that calls the +bridge/control API directly. Uses the app's normal login; nothing new is exposed. This is +the rejected stdio-MCP design's replacement: a server-side tool that talks to the bridge +needs either an internal token (auth-bypass) or to run in-process. **Decision pending (user).** + +### Option B — browser-only flow (works today) + +No new server tool. An external agent drives an open editor purely through the bridge +endpoints using the logged-in owner's session. Restricted to the editor the user has open. +This is the flow demonstrated above and requires **no** auth-bypass. + +A stdio `photo_editor` MCP server was prototyped and **rejected** because the built-in MCP +subprocess has no internal token; wiring it would require expanding an auth bypass. +The two viable options above are left for the user to choose. + +--- + +## 6. Known limitations / honest grading + +Live run against real 7011 with the server LLM planner (7 prompts): all 7 executed +(`planner:"llm"`). Graded by eye: + +- PASS: black & white portrait; warm cinematic; sharper + more vivid; text title. +- FAIL (region-masked, SAM-bound): "dramatic sky" (global over-saturation instead of a + dramatic, masked sky), "blur the background" (coarse mask leaks; visible rectangle), + "brighten the face" (coarse ellipse/rect artifact). + +Whole-image edits are reliable; subject/face/sky region edits need SAM. The self-check +loop works (revert-and-replan) but the region cases spend all 3 iterations. + +## 7. Theming (control kit) + +The editor's interactive controls (ranges, number fields, selects, checkboxes) share a +single themed kit: `static/js/editor/control-kit.js` adds `.ge-ctl-*` classes scoped to +the editor roots (`#gallery-editor-container`, `.ge-adj-popup`, `.ge-filter-overlay`, +`.ge-canvas-size-overlay`, `.ge-float-window`, `.ge-transform-popup`), and +`static/css/13-image-editor.css` (section "Editor control kit") styles them from the app +design tokens (`static/css/00-tokens.css`) to match the app sliders/selects. It keeps +range fills in sync and attaches a **scrub-on-label** affordance to numeric fields +(drag the field's label to change the value; arrow keys work, Shift = x10). + +**The PhotoCraft editor is dark-only.** `:root.light` flips the global tokens, but +`13-image-editor.css` has no `.light` handling and the editor relies on dark surfaces, +so light mode renders a garbled mix. Unlike the rest of Odysseus, the editor follows +Photoshop and is not offered in a light theme. diff --git a/routes/editor_agent_routes.py b/routes/editor_agent_routes.py new file mode 100644 index 000000000..410bab501 --- /dev/null +++ b/routes/editor_agent_routes.py @@ -0,0 +1,1818 @@ +"""Editor agent routes: natural-language planning for the image editor. + +Provides: + POST /api/editor/plan -> {reply, planner, steps:[{op,params}], done?} + POST /api/editor/bridge/command -> enqueue validated ops for the open editor + GET /api/editor/bridge/events -> SSE stream of queued commands (browser subscriber) + POST /api/editor/bridge/result -> browser posts results/screenshots back + POST /api/editor/bridge/await -> wait for a queued command's result + +The browser editor (static/js/editor/nl-edit.js) posts {prompt, state, mode, screenshot} +and consumes {reply, planner, steps} or {done:true}. When mode == "check" and the model +reports completion, the route returns {"done": true} to stop the client's self-check loop. + +The planner is wired to the app's configured chat model (vision-capable when a screenshot +is supplied) via set_editor_planner(); a deterministic keyword planner is the fallback so +the editor always gets a usable plan. +""" +from __future__ import annotations + +import asyncio +import inspect +import json +import logging +import re +import uuid +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from src import editor_style_guide as style_guide +from src import editor_agent_model as editor_agent_model +from src import shutdown as _shutdown + +logger = logging.getLogger(__name__) + +# ── Op catalog (kept in sync with static/js/editor/editor-control.js) ── +OP_CATALOG = [ + "document.info", "document.importImage", + "document.crop", "document.resizeCanvas", "document.savePsd", + "layers.list", "layers.select", "layers.addRaster", "layers.placeImage", "layers.addAdjustment", + "layers.duplicate", "layers.delete", "layers.setOpacity", "layers.setBlend", + "layers.group", "layers.ungroup", "layers.addStyle", + "ai.generativeExpand", + "selection.selectAll", "selection.deselect", "selection.invert", + "selection.rectangle", "selection.ellipse", + "selection.subject", "selection.sky", "selection.background", "selection.samText", + "selection.colorRange", "selection.maskBounds", "selection.exportMask", + "adjustments.apply", "adjustments.applyMasked", + "filters.apply", "filters.applyMasked", + "transform.start", "transform.apply", "transform.cancel", + "text.add", + "history.mark", "history.collapse", "history.undo", "history.redo", + "export.png", "export.gallery", "screenshot", +] +# Retouch/selection ops are defined once in the shared style guide so the +# server planner and the browser fallback cannot drift; a plan naming one of +# these would otherwise be filtered out of the catalog. +for _extra_op in style_guide.NEW_OP_NAMES: + if _extra_op not in OP_CATALOG: + OP_CATALOG.append(_extra_op) +del _extra_op +_OP_SET = frozenset(OP_CATALOG) + +# Canonical argument shapes advertised to the model so it stops guessing. The +# browser client tolerates loose forms as a safety net, but the model should +# emit exactly these. +_ADJUSTMENT_TYPES = ( + "brightness-contrast {brightness,contrast} (multipliers, 1=neutral), " + "exposure {exposure,offset,gamma}, white-balance {temperature,tint}, " + "hue-saturation {hue,saturation,lightness}, vibrance {vibrance}, black-white, " + "shadows-highlights {shadows,highlights}, levels, curves, color-balance, " + "photo-filter {color,density,preserve_luminosity}, selective-color, gradient-map, " + "invert, posterize, threshold, channel-mixer, color-lookup" +) +_FILTER_IDS = ( + "blur-gaussian, blur-motion, blur-zoom, sharpen, sharpen-edges, unsharp-mask, " + "noise-add, noise-despeckle, pixelate-mosaic, pixelate-crystallize, stylize-emboss, " + "stylize-find-edges, stylize-solarize, distort-twirl, distort-pinch, distort-spherize, " + "distort-ripple, render-clouds, clarity {amount,radius}, vignette {amount,feather}" +) +_CANONICAL_GUIDANCE = ( + "Canonical op shapes (use exactly these, never invent keys or names): " + "adjustments.apply / adjustments.applyMasked params = " + "{\"type\": , \"params\": {...}}; " + "filters.apply / filters.applyMasked params = {\"id\": }. " + "Never nest a numeric bag under {\"adjustments\": {...}} with no type, and never invent " + "adjustment types or filter ids. \"clarity\" (params amount,radius) and \"vignette\" " + "(params amount,feather) are real filters — use them for local-contrast and framing. " + "Canvas: document.crop {x,y,width,height}, document.resizeCanvas {width,height,anchor}, " + "ai.generativeExpand {width,height,anchor,prompt,mode:'instruction',feather}. Text/styles: " + "text.add {text,x,y,color,size}, layers.addStyle {type,params}. Export: document.savePsd. " + "ai_edit_image takes a mode: 'instruction' (default; whole image, no mask, prompt names " + "what stays unchanged) for background/scene/season/style changes, adding an object to an " + "existing photo, and removing diffuse distractions; 'masked' (tight selection first) ONLY " + "for explicit regional removal of a clearly identified object. generate_image is ONLY for " + "creating a brand-new image." +) +# The shared style guide is the single source of truth for intent routing, +# calibrated strengths and the low-key/temperature/portrait rules. Both the +# server planner and the browser fallback read it (the client fetches +# GET /api/editor/style-guide). +_STYLE_GUIDANCE = style_guide.guidance() + +# EDITOR_PLANNER: async (messages: list[dict], owner: Optional[str]) -> str | dict. Set by app.py. +_EDITOR_PLANNER: Optional[Callable[[List[dict], Optional[str]], Any]] = None +_MAX_STEPS = 32 +_QUEUE_MAX = 64 +# Generative editor steps (SAM mask + local diffusion) take 30-110s each, so +# bridge waits and the agent's per-step wait must outlast the provider. +_BRIDGE_DEFAULT_TIMEOUT = 330.0 + +# ── Strict JSON ops schema (structured outputs / grammar) ── +# The planner (the local Qwen3.5-9B at :8097 and every fallback) is constrained +# to this schema so it can only emit valid editor ops. Allowed ops: +# * the deterministic editor ops (OP_CATALOG), +# * generate_edit -> image model (instruction, optional mask) [=ai_edit_image] +# * select_object -> SAM mask by description (seneca /v1/images/mask) +# [=selection.samText] +# * ask_user -> ask 2-3 concrete choices (never a silent tone default). +# The model-facing aliases are translated to the internal execution ops below, +# so the browser client and the executor keep their existing op vocabulary. +_MODEL_PLAN_OPS = tuple(dict.fromkeys( + list(OP_CATALOG) + ["generate_edit", "select_object", "ai_edit_image", + "generate_image", "ask_user"] +)) +_ALLOWED_PLAN_OPS = frozenset(OP_CATALOG) | {"ai_edit_image", "generate_image"} +_PLAN_OP_ALIASES = { + "generate_edit": "ai_edit_image", # image model: instruction + optional mask + "select_object": "selection.samText", # SAM mask by description +} +# Inner op values the ai_edit_image tool accepts (src/agent_tools/editor_tools.py +# `_AI_OPS`). A model-authored ai_edit_image step frequently omits the inner op or +# invents one (e.g. "restyle"), which the tool then rejects. Normalise it so +# instruction/restyle edits become op "generative" and masked region edits become +# "inpaint". +_AI_IMAGE_INNER_OPS = frozenset( + {"remove-bg", "upscale", "denoise", "enhance-face", "inpaint", "generative"} +) + + +def _normalize_ai_step(params: Any) -> Dict[str, Any]: + """Repair an ai_edit_image step's inner op so it always runs (restyle -> + generative, masked -> inpaint).""" + params = dict(params) if isinstance(params, dict) else {} + mode = str(params.get("mode") or "").strip().lower() + inner = str(params.get("op") or "").strip().lower() + if inner not in _AI_IMAGE_INNER_OPS: + inner = "inpaint" if mode == "masked" else "generative" + params["op"] = inner + if inner in ("inpaint", "generative") and not mode: + params["mode"] = "masked" if inner == "inpaint" else "instruction" + return params +PLAN_SCHEMA: Dict[str, Any] = { + "type": "object", + "properties": { + "reply": {"type": "string"}, + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": list(_MODEL_PLAN_OPS), + "description": ( + "generate_edit = AI image model edit (params {prompt, " + "mode:'instruction'|'masked'}; do NOT add an inner 'op' — " + "instruction/restyle edits run as the image model's " + "'generative' op, masked region edits as 'inpaint'). " + "select_object = SAM mask by text (params {text}). " + "ask_user = ask with 2-3 choices (params {question, options})." + ), + }, + "params": {"type": "object"}, + }, + "required": ["op"], + }, + }, + "ask_user": { + "type": "object", + "properties": { + "question": {"type": "string"}, + "options": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["question", "options"], + }, + "done": {"type": "boolean"}, + }, + "required": ["reply", "steps"], +} +PLAN_RESPONSE_FORMAT: Dict[str, Any] = { + "type": "json_schema", + "json_schema": {"name": "editor_plan", "schema": PLAN_SCHEMA}, +} + + +def _normalize_ask_user(payload: Any) -> Optional[Dict[str, Any]]: + """Validate an ask_user payload: one question and 2-3 concrete choices.""" + if not isinstance(payload, dict): + return None + question = str(payload.get("question") or "").strip() + raw = payload.get("options") + labels: List[str] = [] + for opt in (raw or []): + if isinstance(opt, str): + lbl = opt.strip() + elif isinstance(opt, dict): + lbl = str(opt.get("label") or "").strip() + else: + lbl = "" + if lbl: + labels.append(lbl) + labels = labels[:3] + if not question or len(labels) < 2: + return None + # Match the shape the ask_user tool + the editor client expect: option + # objects with a label (and an optional description). + return { + "question": question, + "options": [{"label": lbl, "description": ""} for lbl in labels], + "multi": False, + } + + +def _validate_plan(data: Any) -> Optional[Dict[str, Any]]: + """Validate a raw planner plan against the strict ops schema. + + Returns a normalized plan (aliases translated, ops checked against the + allowed set, ask_user normalized to 2-3 choices) or None when the plan is + invalid or empty. Callers turn None into an ask_user plan — never a silent + tone default. + """ + if not isinstance(data, dict): + return None + reply = str(data.get("reply") or "") + ask_user = _normalize_ask_user(data.get("ask_user")) + raw_steps = data.get("steps") + if raw_steps is None: + raw_steps = [] + if not isinstance(raw_steps, list): + return None + steps: List[Dict[str, Any]] = [] + for s in raw_steps[:_MAX_STEPS]: + if not isinstance(s, dict): + continue # drop a malformed step rather than rejecting the whole plan + op = str(s.get("op") or "") + if op == "ask_user": + au = _normalize_ask_user(s.get("params")) + if au is not None and ask_user is None: + ask_user = au + continue + op = _PLAN_OP_ALIASES.get(op, op) + if op not in _ALLOWED_PLAN_OPS: + continue # drop a single unknown/hallucinated op; keep the valid steps + params = s.get("params") if isinstance(s.get("params"), dict) else {} + if op == "ai_edit_image": + params = _normalize_ai_step(params) + steps.append({"op": op, "params": params}) + # Guardrails apply to model plans too (no stacked lifts, no brighten after + # retouch.skin, no global adjustment after a masked recolour). + steps = style_guide.enforce_guards(steps) + done = bool(data.get("done")) + if not steps and not done and ask_user is None: + return None + out: Dict[str, Any] = {"reply": reply, "planner": "llm", "steps": steps} + if ask_user is not None: + out["ask_user"] = ask_user + if done: + out["done"] = True + return out + +# Generative edits (remove/add/replace content) require an image provider. When +# none is available the agent must stop and say so plainly rather than apply a +# deterministic approximation that silently does the wrong thing. +try: + from src.ai_interaction import IMAGE_PROVIDER_REQUIRED_MESSAGE as _PROVIDER_REQUIRED_MESSAGE +except Exception: # pragma: no cover - defensive + _PROVIDER_REQUIRED_MESSAGE = ( + "Removing/adding objects needs an image generation provider; enable one in " + "Settings > Models." + ) + +_AI_CONTENT_OPS = frozenset({"ai_edit_image", "generate_image"}) + +# Model ids that are editing/inpainting models rather than text-to-image ones. +_EDIT_MODEL_RE = re.compile(r"edit|inpaint|fill|kontext|controlnet|removal|qwen-image-edit", re.I) + +# Content nouns that signal a generative object edit rather than a tone/filter +# tweak. Kept narrow so "make it black and white" / "add a vignette" stay +# deterministic. +_GEN_OBJECTS = ( + r"(?:persons?|people|men|man|women|woman|guys?|girls?|boys?|someone|somebody|" + r"objects?|car|cars|sign|signs|logos?|watermarks?|trees?|buildings?|" + r"reflections?|animals?|dogs?|cats?|birds?|hands?|fingers?|moons?|suns?|clouds?|" + r"stars?|fire|smoke|snow|rain|flowers?|mountains?|planets?|balloons?|rockets?|hats?|" + r"glasses|sunglasses|shades|beard|beards|stickers?|frames?|borders?|" + r"rainbows?|wildflowers?|sunflowers?|tulips?|daisies?|leaves|pumpkins?|lanterns?|" + r"butterflies?|bees?|boats?|sailboats?|kites?|umbrellas?)" +) +_REMOVE_RE = re.compile( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out|photoshop\s+out|paint\s+out)\b" + r"[^.!?\n]{0,60}?\b" + _GEN_OBJECTS + r"\b", + re.I, +) +_ADD_RE = re.compile( + r"\b(?:add|insert|put|place|draw|paint|create|generate)\b" + r"[^.!?\n]{0,40}?\b" + _GEN_OBJECTS + r"\b", + re.I, +) +_REPLACE_RE = re.compile( + r"\b(?:replace|swap|substitute)\b|\b(?:change|turn)\b[^.!?\n]{0,40}?\b(?:to|into)\b", + re.I, +) +# Removing diffuse clutter ("remove the distractions") is generative too, but +# the specific-noun _REMOVE_RE above does not catch it. +_REMOVE_CLEANUP_RE = re.compile( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out|clean\s+up|declutter|clear)\b" + r"[^.!?\n]{0,40}?\b(?:distractions?|clutter|junk|mess|noise|" + r"unwanted\s+(?:objects?|items?|things?)|stuff|things|blemishes?|spots?)\b", + re.I, +) + +# Changing the whole background/scene (or placing the subject somewhere new) +# is a generative replacement, not the local remove-bg op. "make the +# background nicer/better" is included; plain "blur the background" is not. +_BG_RE = re.compile( + r"\b(?:replace|change|swap|put|place|set|make|improve|enhance|beautify|upgrade)\b" + r"[^.!?\n]{0,40}?\b(?:background|backdrop|scene)\b" + r"|\b(?:put|place)\b[^.!?\n]{0,30}?\b(?:on|in|at)\s+(?:a|an|the)\s+" + r"(?:beach|street|mountains?|forest|city|park|ocean|sea|desert|room|stage|field|snow|" + r"night|moon|mars|space|island|jungle)\b", + re.I, +) +# Season/weather changes and an explicit "generative fill" change scene +# content (not just tone), so they must be a generative edit, never a +# colour adjustment/filter step. +_SEASON_WEATHER_RE = re.compile( + r"\b(?:snow(?:y|ing|fall)?|winter|autumn(?:al)?|fall\s+(?:leaves|foliage)|spring|" + r"rain(?:y|ing)?|fog(?:gy)?|mist(?:y)?|storm(?:y)?|overcast)\b", + re.I, +) +_GENERATIVE_FILL_RE = re.compile(r"\bgenerative(?:\s+fill)?\b", re.I) +# Whole-image artistic RESTYLE requests (turn the photo into a painting, anime, +# pixel art, ...). These are a model instruction edit over the whole frame, not a +# local adjustment. The client has a short fast-path regex; the server must +# generalise so unknown restyle phrasings still reach the model. +_RESTYLE_RE = re.compile( + r"\b(?:re[\s-]?style|painting|painterly|water[\s-]?colou?r|oil\s+paint(?:ing)?|" + r"acrylic|gouache|pastel|sketch|pencil|charcoal|ink\s+drawing|line\s+art|comic|" + r"cartoon|anime|manga|illustration|3d\s+render|cgi|claymation|clay|pixel\s+art|" + r"voxel|low[\s-]?poly|mosaic|stained\s+glass|pop\s+art|art\s+nouveau|ukiyo[\s-]?e|" + r"surreal|fantasy\s+art|steampunk|cyberpunk|vaporwave|synthwave|graffiti|woodcut|" + r"linocut|etching|lithograph|impressionist|cubist|abstract|doodle|vector\s+art|" + r"neon\s+sign|psychedelic|gothic|comic[\s-]?book|storybook)\b", + re.I, +) + +# Text/title/PSD/format requests are deterministic edits, never generative. +_TEXT_RE = re.compile( + r"\b(?:title|caption|subtitle|label|heading|watermark|text|word(?:ing)?)\b", re.I, +) +_TEXT_ADD_RE = re.compile(r"\b(?:add|insert|put|place|write|overlay|create|set)\b", re.I) +_STYLE_RE = re.compile( + r"\b(drop[\s-]?shadow|outer[\s-]?glow|glow|stroke|outline|emboss|bevel)\b", re.I, +) +_PSD_RE = re.compile(r"\b(?:psd|photoshop file|layered file)\b", re.I) + +# Aspect -> exact output size. Instagram/format intents crop to the aspect and +# then resize to this size (then optionally add a title). +_ASPECT_TARGETS = ( + (re.compile(r"\b(?:1080\s*[x×]\s*1350|4\s*:\s*5|four[\s-]?fifths?|instagram)\b", re.I), (1080, 1350)), + (re.compile(r"\b(?:1080\s*[x×]\s*1080|1\s*:\s*1|square)\b", re.I), (1080, 1080)), + (re.compile(r"\b(?:1080\s*[x×]\s*1920|9\s*:\s*16|story|reel|tiktok|vertical)\b", re.I), (1080, 1920)), + (re.compile(r"\b(?:1920\s*[x×]\s*1080|16\s*:\s*9|widescreen|landscape video)\b", re.I), (1920, 1080)), +) +_DIM_RE = re.compile(r"\b(\d{3,5})\s*[x×]\s*(\d{3,5})\b") + + +def _is_text_request(text: str) -> bool: + return bool(_TEXT_RE.search(text) and _TEXT_ADD_RE.search(text)) + + +def _format_target(text: str) -> Optional[Tuple[int, int]]: + for pattern, dims in _ASPECT_TARGETS: + if pattern.search(text): + return dims + m = _DIM_RE.search(text) + if m: + try: + return (int(m.group(1)), int(m.group(2))) + except Exception: + return None + return None + + +def _extend_requested(text: str) -> bool: + if re.search(r"\b(?:extend|expand|wider|taller|letterbox)\b", text, re.I): + return True + return bool(re.search( + r"\badd\b[^.!?\n]{0,25}?\b(?:canvas|border|space|margin)\b", text, re.I, + )) + + +def _generative_kind(prompt: str) -> Optional[str]: + """Classify a prompt as a generative content edit: 'remove'/'add'/'replace'. + + Returns None for tone/colour/filter/text/selection edits that deterministic + editor ops handle correctly. 'remove the background' is deliberately not + matched (that is the local remove-bg op). + """ + text = str(prompt or "") + if not text.strip(): + return None + # Text/title, PSD and format requests are deterministic — never generative. + if _is_text_request(text) or _PSD_RE.search(text) or _format_target(text): + return None + # An explicit brand-new image ("create/generate/make a new image/picture/logo + # ...") is a text-to-image generation, never an edit of the current photo. + if _is_new_image_request(text): + return "add" + # Replacing the background/scene (or putting the subject somewhere new) is + # a generative replacement even though it is phrased like a background op. + if _BG_RE.search(text): + return "replace" + if re.search( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of)\s+(?:the\s+)?background\b", + text, + re.I, + ): + return None + # Season/weather changes and explicit "generative fill" change content, not + # tone. Only treat them as generative when the user is not asking to REMOVE + # that element ("remove the rain" stays a removal). + if _GENERATIVE_FILL_RE.search(text) or _SEASON_WEATHER_RE.search(text): + if not re.search( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out)\b", text, re.I + ): + return "replace" + if _RESTYLE_RE.search(text) and not _is_new_image_request(text): + # A whole-image artistic restyle is a model instruction edit (keep the + # subject/composition), never a local tone/colour adjustment. + return "restyle" + if _REPLACE_RE.search(text): + return "replace" + if _ADD_RE.search(text): + return "add" + if _REMOVE_RE.search(text) or _REMOVE_CLEANUP_RE.search(text): + return "remove" + return None + + +_NEW_IMAGE_RE = re.compile( + r"\b(?:create|generate|make|produce|draw|design)\b[^.!?\n]{0,30}?\b" + r"(?:new\s+)?(?:image|picture|photo|illustration|artwork|graphic|logo|icon|wallpaper|poster)\b", + re.I, +) +_EDIT_CONTEXT_RE = re.compile( + r"\b(?:this|that|these|it|the\s+(?:photo|image|picture)|attached|from\s+this|" + r"of\s+(?:this|the\s+photo)|edit|in\s+(?:this|the)\s+photo)\b", + re.I, +) +_KEEP_CLAUSE_RE = re.compile(r"\bkeep\b|\bunchanged\b|\bpreserve\b|\bsame\b|\bidentical\b", re.I) + + +def _is_new_image_request(text: str) -> bool: + """True only for a from-scratch image generation (not an edit of an existing photo).""" + if not _NEW_IMAGE_RE.search(text): + return False + return not _EDIT_CONTEXT_RE.search(text) + + +def _instruction_prompt(text: str) -> str: + """Append a keep-unchanged clause so instruction edits preserve the subject.""" + t = str(text or "").strip() + if not t: + return t + if _KEEP_CLAUSE_RE.search(t): + return t + return ( + t.rstrip(".") + + ". Keep the main subject and everything else in the image exactly the same." + ) + + +def _restyle_prompt(text: str) -> str: + """Whole-image artistic restyle instruction (subject/composition preserved).""" + t = str(text or "").strip() + if t: + base = ( + "Restyle the entire image as requested, applying the artistic style " + "across the whole frame: " + t + ) + else: + base = "Restyle the entire image in a new artistic style." + return _instruction_prompt(base) + + +def _generative_mode(kind: str, prompt: str) -> str: + """'masked' only for an explicit regional removal of a clearly identified object. + + Everything else generative (background/season/style replacement, adding an + object to the scene, removing diffuse distractions) uses 'instruction'. + """ + if kind == "remove": + text = str(prompt or "") + if _REMOVE_RE.search(text) and not _REMOVE_CLEANUP_RE.search(text): + return "masked" + return "instruction" + + +def _removal_prompt(text: str) -> str: + """A more explicit diffuse-removal instruction that names common clutter.""" + t = str(text or "").strip() or "Remove the distracting clutter from the photo" + return ( + t.rstrip(". ") + + ". Remove any distracting clutter such as roads, guardrails, utility poles, " + "signs, wires and debris, and fill the cleared area with the surrounding " + "landscape so it looks natural." + ) + + +# Rich, scene-specific descriptions so a background replacement produces a +# convincing scene (Qwen-Image-Edit is an instruction model and a bare "a beach" +# tends to yield a flat blue backdrop). Keyed by keywords in the request. +_SCENE_HINTS = ( + (r"beach|shore|seaside|coast|tropical", + "a sunny tropical beach", + "golden sand, turquoise sea with gentle waves, a clear blue sky with soft " + "clouds, and natural daylight"), + (r"mountain|alps|peak|valley", + "a scenic mountain landscape", + "distant peaks, natural daylight and soft haze"), + (r"forest|woods|jungle", + "a lush green forest", + "dappled natural daylight and depth"), + (r"city|street|urban|downtown", + "a realistic city street", + "recognisable city buildings and depth of field with natural perspective"), + (r"field|meadow|grass", + "an open grassy field", + "natural daylight and a distant horizon"), + (r"snow|winter", + "a snow-covered landscape", + "cold, soft daylight and a pale sky"), + (r"desert|dunes", + "a warm desert", + "sand dunes and a clear sky"), + (r"studio|plain|neutral|solid", + "a clean neutral studio backdrop", + "soft, even lighting"), +) +_SCENE_DEFAULT = ( + "a realistic natural background", + "matching the subject's lighting, perspective and shadows", +) +# Season/weather scene descriptions ("make it snowy/autumn"). +_SEASON_HINTS = ( + (r"snow|winter|snowy", "a snowy winter scene", + "fresh snow covering the ground and surfaces, soft cold daylight, a pale overcast sky"), + (r"autumn|autumnal|fall", "an autumn scene", + "vivid orange and golden foliage, warm low sunlight and fallen leaves"), + (r"spring", "a bright spring scene", + "fresh green foliage and blossoms under bright natural daylight"), + (r"rain|storm|overcast", "a rainy overcast scene", + "wet reflective surfaces, raindrops and a dramatic overcast sky"), + (r"fog|mist", "a foggy scene", + "soft drifting mist and diffuse light"), + (r"sky|clouds", "a new sky", + "realistic clouds and natural lighting matching the scene"), +) + + +def _scene_prompt(text: str) -> str: + """Deterministic, templated instruction for a background/scene change. + + Qwen-Image-Edit is an instruction model and a bare scene noun tends to yield + a flat backdrop, so the scene is expanded into concrete detail. The result is + passed to the provider unmodified. + """ + t = str(text or "").strip() + if not t: + return t + if _BG_RE.search(t): + scene, details = _SCENE_DEFAULT + for pattern, scene_name, description in _SCENE_HINTS: + if re.search(pattern, t, re.I): + scene, details = scene_name, description + break + return ( + f"Replace the entire background behind the person with {scene}: {details}. " + "Keep the person exactly the same." + ) + if _SEASON_WEATHER_RE.search(t) or _GENERATIVE_FILL_RE.search(t): + scene, details = _SCENE_DEFAULT + for pattern, scene_name, description in _SEASON_HINTS: + if re.search(pattern, t, re.I): + scene, details = scene_name, description + break + return ( + f"Change the scene to {scene}: {details}. " + "Keep the main subject and everything else in the image exactly the same." + ) + return t + + +def _generative_step(kind: str, prompt: str) -> Dict[str, Any]: + """The AI step a generative intent must use instead of deterministic ops.""" + instruction = str(prompt or "").strip() + if kind == "restyle": + return { + "op": "ai_edit_image", + "params": {"op": "generative", "mode": "instruction", + "prompt": _restyle_prompt(instruction), "steps": 8}, + } + if kind == "add": + # A brand-new image is the only case for generate_image; adding an object + # to an existing photo is an instruction edit of the whole image. + if _is_new_image_request(prompt): + return {"op": "generate_image", "params": {"prompt": instruction}} + return { + "op": "ai_edit_image", + "params": { + "op": "generative", + "mode": "instruction", + "prompt": _instruction_prompt(instruction), + "steps": 8, + }, + } + mode = _generative_mode(kind, prompt) + op = "inpaint" if kind == "remove" else "generative" + if mode == "instruction": + if kind == "remove": + base = _removal_prompt(instruction) + else: + base = _scene_prompt(instruction) + params: Dict[str, Any] = {"op": op, "mode": "instruction", "prompt": _instruction_prompt(base)} + else: + params = {"op": op, "mode": "masked", "prompt": instruction} + if kind in ("remove", "replace"): + # Removal and background-replacement edits need more sampler steps than + # the server's default of 4. + params["steps"] = 8 + return {"op": "ai_edit_image", "params": params} + + +_SKY_ADD_RE = re.compile( + r"\b(?:sky|moons?|suns?|sunset|sunrise|clouds?|stars?|night|aurora|stars?)\b", + re.I, +) +_REMOVE_NOUN_RE = re.compile( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out|clean\s+up|declutter|clear)\b" + r"[^.!?\n]{0,60}?\b(" + + _GEN_OBJECTS + + r"|distractions?|clutter|junk|mess|noise|stuff|things)\b", + re.I, +) + + +def _selection_step_for(kind: str, prompt: str) -> Optional[Dict[str, Any]]: + """A deterministic selection step that yields the mask a MASKED ai_edit_image needs. + + Instruction-mode edits operate on the whole image and need no mask, so this + returns None for them. Masked mode is only used for an explicit regional + object removal; the selection targets that object so ``selection.exportMask`` + has a tight region to export. + """ + text = str(prompt or "") + if kind == "remove" and _generative_mode(kind, text) == "masked": + m = _REMOVE_NOUN_RE.search(text) + label = (m.group(1).lower() if m else "person") + if label.endswith("s"): + label = label[:-1] + if label in ("people",): + label = "person" + return {"op": "selection.subject", "params": {"label": label}} + return None + + +def _generative_steps(kind: str, prompt: str) -> List[Dict[str, Any]]: + """The selection step (when needed) followed by the AI content step.""" + steps: List[Dict[str, Any]] = [] + selection = _selection_step_for(kind, prompt) + if selection: + steps.append(selection) + steps.append(_generative_step(kind, prompt)) + return steps + + +def _plan_uses_ai_content(steps: Any) -> bool: + return any(isinstance(s, dict) and s.get("op") in _AI_CONTENT_OPS for s in (steps or [])) + + +def _ensure_generative(prompt: str, steps: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Force a generative intent onto an AI content op, never a substitute. + + If the prompt is a generative content edit (remove/add/replace) but the + plan has no ai_edit_image/generate_image step, replace the plan with the + correct AI step so the agent cannot silently do an unrelated deterministic + edit. Non-generative prompts are returned unchanged. + """ + kind = _generative_kind(prompt) + if not kind: + return steps + # Background/scene replacement is deterministic: always use the templated + # scene prompt so a model rewrite cannot drop the scene details (a model + # ai_edit_image step would otherwise pass with its own vague prompt). + if kind == "replace" and ( + _BG_RE.search(str(prompt or "")) + or _SEASON_WEATHER_RE.search(str(prompt or "")) + or _GENERATIVE_FILL_RE.search(str(prompt or "")) + ): + return _generative_steps(kind, prompt) + # For a restyle (and any other generative intent) the planner is expected to + # choose generate_edit itself; the _RESTYLE_RE keyword list is only a fast + # path used when the model did not already pick an AI content op. + if not _plan_uses_ai_content(steps): + return _generative_steps(kind, prompt) + return steps + + +def _image_provider_available(owner: Optional[str]) -> bool: + try: + from src.ai_interaction import image_editing_provider_available + + return bool(image_editing_provider_available(owner)) + except Exception: + return True # unknown: let the tool attempt and report its own error + + +def _state_dims(state: Any) -> Optional[Tuple[int, int]]: + """Current canvas size from the client-sent state (flat or nested).""" + if not isinstance(state, dict): + return None + candidates = [state] + for key in ("document", "document_info", "info"): + if isinstance(state.get(key), dict): + candidates.append(state[key]) + for src in candidates: + w = src.get("width") or src.get("imgWidth") + h = src.get("height") or src.get("imgHeight") + try: + if w and h: + return (int(round(float(w))), int(round(float(h)))) + except Exception: + continue + return None + + +def _title_text(prompt: str) -> Optional[str]: + quoted = re.search(r"[\"'\u201c\u201d\u2018\u2019]([^\"'\u201c\u201d\u2018\u2019]{1,40})[\"'\u201c\u201d\u2018\u2019]", prompt) + if quoted and quoted.group(1).strip(): + return quoted.group(1).strip() + caps = re.search(r"\b([A-Z][A-Z0-9]{1,20})\b", prompt) + if caps and caps.group(1).lower() not in {"add", "title", "text", "psd"}: + return caps.group(1) + return None + + +def _crop_rect_for_aspect(cur: Tuple[int, int], target: Tuple[int, int]) -> Optional[Dict[str, int]]: + cw, ch = cur + tw, th = target + if cw <= 0 or ch <= 0 or tw <= 0 or th <= 0: + return None + target_ratio = tw / th + cur_ratio = cw / ch + if abs(cur_ratio - target_ratio) < 0.02: + return None # already the right aspect + if cur_ratio > target_ratio: + # too wide -> trim the sides + new_w = int(round(ch * target_ratio)) + new_w = min(cw, max(1, new_w)) + return {"x": (cw - new_w) // 2, "y": 0, "width": new_w, "height": ch} + # too tall -> trim top/bottom + new_h = int(round(cw / target_ratio)) + new_h = min(ch, max(1, new_h)) + return {"x": 0, "y": (ch - new_h) // 2, "width": cw, "height": new_h} + + +def _extend_dims(cur: Tuple[int, int], target: Tuple[int, int]) -> Tuple[int, int]: + """Target canvas that grows (never crops) toward the target aspect.""" + cw, ch = cur + target_ratio = target[0] / target[1] + cur_ratio = cw / ch + if abs(cur_ratio - target_ratio) < 0.02: + return (cw, ch) + if cur_ratio < target_ratio: + return (int(round(ch * target_ratio)), ch) # add width + return (cw, int(round(cw / target_ratio))) # add height + + +def _plan_text_steps(prompt: str) -> List[Dict[str, Any]]: + title = _title_text(prompt) or "Title" + steps: List[Dict[str, Any]] = [{ + "op": "text.add", + "params": {"text": title, "x": 40, "y": 40, "color": "#ffffff", "size": 64}, + }] + if _STYLE_RE.search(prompt): + steps.append({"op": "layers.addStyle", "params": {"type": "drop-shadow", "params": {}}}) + return steps + + +def _plan_format_steps(prompt: str, state: Any, provider_available: bool) -> Optional[List[Dict[str, Any]]]: + target = _format_target(prompt) + if not target: + return None + cur = _state_dims(state) + if not cur: + # No canvas size known: ask the client to crop to the aspect itself. + return None + steps: List[Dict[str, Any]] = [] + if _extend_requested(prompt): + new_w, new_h = _extend_dims(cur, target) + if (new_w, new_h) == cur: + return None + if provider_available: + steps.append({"op": "ai.generativeExpand", "params": { + "width": new_w, "height": new_h, + "anchor": "center", "mode": "instruction", "feather": True, + "steps": 8, + "prompt": "extend the surrounding scene and sky naturally into the new border, " + "keep the original photo unchanged"}}) + else: + steps.append({"op": "document.resizeCanvas", "params": { + "width": new_w, "height": new_h, "anchor": "center"}}) + else: + rect = _crop_rect_for_aspect(cur, target) + if rect: + steps.append({"op": "document.crop", "params": rect}) + # Only resize when an exact output size was requested. + if abs(target[0] / target[1] - cur[0] / cur[1]) >= 0.02 or target in ((1080, 1350), (1080, 1080), (1080, 1920), (1920, 1080)): + steps.append({"op": "document.resizeCanvas", "params": { + "width": target[0], "height": target[1], "anchor": "center"}}) + if not steps: + return None + # Optional caption/title that came with the format request. + if _is_text_request(prompt): + steps.extend(_plan_text_steps(prompt)) + return steps + + +def _plan_psd_steps(prompt: str) -> Optional[List[Dict[str, Any]]]: + if _PSD_RE.search(prompt): + return [{"op": "document.savePsd", "params": {}}] + return None + + +def _plan_special( + prompt: str, state: Any, provider_available: bool, +) -> Optional[Tuple[List[Dict[str, Any]], str, str]]: + """Deterministic overrides for intents the model reliably botches. + + Canvas format/crop/extend, text + layer styles and PSD export must not be + left to the model. Returns (steps, reply, note) or None. These never need an + image provider except generative border fill, used only when one exists. + """ + # Targeted recolour is fully deterministic (object mask + masked hue shift) + # and must never be left to the model, which sometimes substitutes a + # generative placeImage. Highest priority so it can never vary. + recolor = style_guide.recolor_plan(prompt) + if recolor: + return list(recolor["steps"]), str(recolor["reply"]), "" + if _format_target(prompt): + steps = _plan_format_steps(prompt, state, provider_available) + if not steps: + return None + target = _format_target(prompt) + reply = f"Setting the canvas to {target[0]}\u00d7{target[1]}." + note = "" + if _extend_requested(prompt) and not provider_available: + note = ("No image generation provider is configured, so the new border " + "was extended with edge/transparent pixels.") + return steps, reply, note + if _PSD_RE.search(prompt): + steps = _plan_psd_steps(prompt) + return (steps, "Saving a layered PSD.", "") if steps else None + if _is_text_request(prompt): + steps = _plan_text_steps(prompt) + if steps: + reply = ("Adding the title with a drop shadow." + if _STYLE_RE.search(prompt) else "Adding the title text.") + return steps, reply, "" + # Structural ops (fill, text edit, align, rotate/flip, copy/paste, flatten, + # merge-down, colour overlay) — the guide's deterministic routing so the + # model never substitutes a generic adjustment/transform for them. + structural = style_guide.structural_plan(prompt) + if structural: + return list(structural["steps"]), str(structural["reply"]), "" + return None + + +_DIMS_CLAIM_RE = re.compile(r"(\d{2,5})\s*[x\u00d7]\s*(\d{2,5})") +_CANVAS_CHANGE_OPS = frozenset({ + "document.crop", "document.resizeCanvas", "ai.generativeExpand", "document.importImage", +}) +_TEXT_OPS = frozenset({"text.add", "layers.addStyle", "document.savePsd"}) + + +def _step_op_signature(step: Any) -> str: + """Identify an executed step by op plus its adjustment type / filter id.""" + if not isinstance(step, dict): + return "" + op = str(step.get("op") or "") + params = step.get("params") or {} + if op in ("adjustments.apply", "adjustments.applyMasked"): + return f"{op}:{params.get('type') or ''}" + if op in ("filters.apply", "filters.applyMasked"): + return f"{op}:{params.get('id') or ''}" + return op + + +# Phrases that name a specific edit. If the reply claims one of these but the +# executed steps do not contain it, the reply is rewritten from the real steps. +_OP_CLAIM_KEYWORDS: List[Tuple[str, Tuple[str, ...]]] = [ + ("photo.autoTone", ("auto tone", "auto-tone", "autotone", "auto-tune")), + ("adjustments.apply:shadows-highlights", ("shadow", "highlight")), + ("adjustments.apply:white-balance", ("white balance", "white-balance", + "temperature", "colour cast", "color cast")), + ("adjustments.apply:vibrance", ("vibrance",)), + ("adjustments.apply:brightness-contrast", ("brightness", "contrast")), + ("adjustments.apply:black-white", ("black and white", "black & white", + "grayscale", "greyscale", "monochrome")), + ("filters.apply:clarity", ("clarity", "local contrast")), + ("filters.apply:vignette", ("vignette",)), + ("filters.apply:sharpen", ("sharpen", "crisper")), + ("retouch.skin", ("skin", "complexion")), + ("retouch.teeth", ("teeth", "smile")), +] + +_OP_LABELS: Dict[str, str] = { + "photo.autoTone": "auto tone", + "adjustments.apply:shadows-highlights": "a shadow lift (highlights protected)", + "adjustments.apply:white-balance": "white balance", + "adjustments.apply:vibrance": "vibrance", + "adjustments.apply:brightness-contrast": "brightness/contrast", + "adjustments.apply:black-white": "black and white", + "filters.apply:clarity": "local contrast (clarity)", + "filters.apply:vignette": "a light vignette", + "filters.apply:sharpen": "sharpening", + "selection.face": "a face selection", + "selection.background": "a background selection", + "retouch.skin": "gentle skin retouching", + "retouch.teeth": "teeth whitening", +} + + +def _reply_from_ops(steps: List[Dict[str, Any]]) -> str: + """Describe what was actually executed, never what was planned.""" + labels: List[str] = [] + for step in steps or []: + sig = _step_op_signature(step) + label = _OP_LABELS.get(sig) + if label is None: + label = sig.split(":")[0] or "an edit" + if label not in labels: + labels.append(label) + if not labels: + return "Done." + return "Done \u2014 I applied " + ", ".join(labels) + "." + + +def _reply_matches_ops(planned_reply: str, steps: List[Dict[str, Any]]) -> bool: + """True when the reply does not claim an edit that was not executed.""" + text = (planned_reply or "").lower() + if not text: + return True + # "drop shadow" is a layer style, not a shadows/highlights adjustment. + text = text.replace("drop shadow", "drop-shadow-style") + executed = {_step_op_signature(step) for step in (steps or [])} + for sig, keywords in _OP_CLAIM_KEYWORDS: + if sig in executed: + continue + if any(keyword in text for keyword in keywords): + return False + return True + + + +def _verified_reply( + prompt: str, + steps: List[Dict[str, Any]], + results: List[Dict[str, Any]], + final_state: Any, + planned_reply: str, +) -> str: + """Build the final reply from ACTUAL executed results + final document state. + + Never let a plan's optimistic prose claim an edit that did not happen. + """ + failed = [ + r for r in (results or []) + if isinstance(r, dict) and r.get("ok") is False + ] + if failed: + op = str(failed[0].get("op") or "step") + err = failed[0].get("error") or "it did not complete" + return (f"I couldn't finish the {op} step ({err}). " + f"Nothing else was left changed — the image is back as it was.") + dims = _state_dims(final_state) + step_ops = [str(s.get("op") or "") for s in (steps or []) if isinstance(s, dict)] + claim = _DIMS_CLAIM_RE.search(planned_reply or "") + if claim and dims: + try: + claimed = (int(claim.group(1)), int(claim.group(2))) + except Exception: + claimed = None + if claimed and (abs(dims[0] - claimed[0]) > 2 or abs(dims[1] - claimed[1]) > 2): + if not any(op in _CANVAS_CHANGE_OPS for op in step_ops): + return (f"I planned to make it {claimed[0]}\u00d7{claimed[1]}, but the canvas is " + f"still {dims[0]}\u00d7{dims[1]} — the resize didn't apply. " + f"Please try again in the editor.") + if not steps and not results: + return planned_reply or "I didn't have a concrete edit to apply." + if not _is_text_request(prompt) and not _reply_matches_ops(planned_reply, steps): + return _reply_from_ops(steps) + if _is_text_request(prompt) and any(op in _TEXT_OPS for op in step_ops) and not failed: + return planned_reply or "Added the text." + if dims and any(op in _CANVAS_CHANGE_OPS for op in step_ops): + return (planned_reply or f"Done — the canvas is now {dims[0]}\u00d7{dims[1]}.") + return planned_reply + + +def set_editor_planner( + fn: Optional[Callable[[List[dict], Optional[str]], Any]], +) -> None: + global _EDITOR_PLANNER + _EDITOR_PLANNER = fn + + +_ROUTING_GUIDANCE = ( + "Routing rules (follow strictly): removing objects/people, adding new content " + "(e.g. a moon, an animal, a sign), or replacing/changing what is in the picture " + "are GENERATIVE edits -> use op \"ai_edit_image\" (op \"inpaint\" to remove, " + "\"generative\" to replace, with a descriptive prompt) or op \"generate_image\" " + "({prompt}) for added content. NEVER approximate these with selection.rectangle, " + "adjustments.apply, filters.apply, transform.* or any other deterministic op. " + "Use deterministic ops only for tone/colour/filter/crop/resize/text/selection " + "edits that do not invent or delete picture content. \"remove the background\" " + "is op \"ai_edit_image\" op \"remove-bg\". If a generative edit is requested and " + "no image provider is configured, do NOT substitute a deterministic edit: return " + "{\"reply\": \"" + _PROVIDER_REQUIRED_MESSAGE + "\", \"steps\": []}." +) + +# Strict ops vocabulary for the model (the schema is also sent as a structured +# output constraint, so the model can only emit these ops). +_SCHEMA_NOTE = ( + "Reply with ONLY a JSON object matching the ops schema. Valid step ops are the " + "deterministic editor ops above plus: \"generate_edit\" (a model image edit; " + "params {\"prompt\": str, \"mode\": \"instruction\"|\"masked\", \"mask\": optional " + "object description}) for any edit that invents, removes, replaces, recolours or " + "restyles image content; \"select_object\" (params {\"text\": object description}) " + "to select an object by description; and \"ask_user\" (params {\"question\": str, " + "\"options\": [2-3 short choices]}) when the request is ambiguous. For " + "\"generate_edit\" set params {\"prompt\": str, \"mode\": \"instruction\"|\"masked\"} " + "and do NOT set an inner \"op\" field: the server maps instruction/restyle edits to " + "the image model's \"generative\" op and masked region edits to \"inpaint\". Prefer " + "generate_edit over deterministic approximations for content changes, and ask_user " + "with 2-3 concrete choices rather than guessing a tone tweak." +) + +_SYSTEM_PROMPT = ( + "You are a photo-editor planning agent. Convert the user's request into a JSON plan of " + "editor operations. Return ONLY JSON: {\"reply\": str, \"steps\": [{\"op\": str, \"params\": {}}]}. " + "Available ops: " + ", ".join(OP_CATALOG) + ", plus ai_edit_image and generate_image. " + + _ROUTING_GUIDANCE + " " + + _CANONICAL_GUIDANCE + " " + _STYLE_GUIDANCE + " " + "Use selection.* before targeted edits when possible. Keep to <=6 steps. " + "If a screenshot is attached, use it to judge the current result. " + + _SCHEMA_NOTE +) + +_CHECK_SYSTEM_PROMPT = ( + "You are a photo-editor quality checker. A screenshot of the editor's CURRENT result is " + "attached along with the user's request and the iteration number. Decide whether the edit " + "satisfies the request. Return ONLY JSON: if it is complete, {\"done\": true, \"reply\": str}; " + "if more work is needed, {\"done\": false, \"reply\": str, \"steps\": [{\"op\": str, \"params\": {}}]}. " + "Available ops: " + ", ".join(OP_CATALOG) + ", plus ai_edit_image and generate_image. " + + _ROUTING_GUIDANCE + " " + + _CANONICAL_GUIDANCE + " " + _STYLE_GUIDANCE + " " + "Never repeat steps that already ran; <=4 steps. " + + _SCHEMA_NOTE +) + + +_GENERIC_TONE_OPS = frozenset({"photo.autoTone", "adjustments.apply", "filters.apply"}) + + +def _plan_is_generic_tone_only(steps: Any) -> bool: + """True when a plan is nothing but whole-image tone/colour/filter steps.""" + items = [s for s in (steps or []) if isinstance(s, dict)] + return bool(items) and all(s.get("op") in _GENERIC_TONE_OPS for s in items) + + +def _unsure_plan(prompt: str, plan: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Demote a bare whole-image tone plan to an ask when the prompt is unsure. + + When the planner produced ONLY generic tone ops (photo.autoTone / + adjustments.apply / filters.apply) for a prompt that matches no concrete + intent and names no concrete defect, the editor is guessing ("it just adds + vibrance and brightness"). Ask the user with 2-3 concrete choices instead. + """ + if not isinstance(plan, dict): + return plan + steps = plan.get("steps") or [] + if not _plan_is_generic_tone_only(steps): + return plan + if _generative_kind(prompt): + return plan + if style_guide.has_concrete_intent(prompt): + return plan + return style_guide.ask_user_plan() + + +def _fallback_plan(prompt: str) -> Dict[str, Any]: + kind = _generative_kind(prompt) + if kind: + label = {"remove": "object removal", "add": "content addition", + "replace": "content replacement", "restyle": "artistic restyle"}[kind] + return { + "reply": f"Planned an AI {label}.", + "planner": "fallback", + "steps": _generative_steps(kind, prompt), + } + guided = style_guide.fallback_plan(prompt) + if guided is not None: + guided["steps"] = style_guide.enforce_guards(guided.get("steps") or []) + return guided + # Nothing mapped to a concrete intent: ASK instead of silently applying a + # default tone tweak. The old fallback emitted a generic + # photo.autoTone/adjustments/filters stack here, which is the "it just adds + # vibrance and brightness" bug for unrecognized prompts. + return style_guide.ask_user_plan() + + +def _parse_plan(raw: Any) -> Optional[Dict[str, Any]]: + if isinstance(raw, dict): + data = raw + else: + text = raw if isinstance(raw, str) else "" + if not text: + return None + try: + data = json.loads(text) + except Exception: + start, end = text.find("{"), text.rfind("}") + if start < 0 or end <= start: + return None + try: + data = json.loads(text[start:end + 1]) + except Exception: + return None + # Every plan is validated against the strict ops schema; an invalid or empty + # plan returns None so the caller falls back to ask_user (never a tone + # default). See _validate_plan. + return _validate_plan(data) + + +def _normalize_steps(body: Dict[str, Any]) -> List[Dict[str, Any]]: + if isinstance(body.get("steps"), list): + raw = body["steps"] + elif body.get("op"): + raw = [{"op": body["op"], "params": body.get("params") or {}}] + else: + raw = [] + out: List[Dict[str, Any]] = [] + for step in raw[:_MAX_STEPS]: + if not isinstance(step, dict): + continue + op = str(step.get("op") or "") + if op not in _OP_SET: + continue + out.append({"op": op, "params": step.get("params") or {}}) + return out + + +def _user(request: Request) -> Optional[str]: + try: + from src.auth_helpers import get_current_user + return get_current_user(request) + except Exception: + return None + + +async def _default_editor_planner(messages: List[dict], owner: Optional[str]) -> Optional[str]: + """Plan with the local editor-agent model, falling back through the chain. + + Order: per-request ``agent_model`` override -> local Qwen3.5-9B (vision, + screenshot attached) -> the same model text-only with a VL caption of the + screenshot -> the hosted default/utility chat model. The model actually + used is recorded so the route can report it. + """ + result = await editor_agent_model.plan_with_editor_agent( + messages, owner, override=editor_agent_model.get_override(), + response_format=PLAN_RESPONSE_FORMAT, + ) + if result is None: + return None + raw, model_used, source = result + editor_agent_model.set_last_model(model_used, source) + return raw + + +def setup_editor_agent_routes() -> APIRouter: + router = APIRouter(tags=["editor-agent"]) + + if _EDITOR_PLANNER is None: + set_editor_planner(_default_editor_planner) + + # Probe the local editor-agent model server (non-blocking) so the planner + # can use it for vision planning when reachable. + editor_agent_model.startup_probe() + + # scoped session -> list of that session's live subscriber queues; id -> result future. + # A command is BROADCAST to every live subscriber (per-connection queue), never to a + # single shared queue: EventSource reconnects overlap, and with one shared queue a + # command could be consumed by a stale subscriber while the active page got nothing. + subscribers: Dict[str, List["asyncio.Queue[dict]"]] = {} + backlog: Dict[str, List[dict]] = {} + pending: Dict[str, asyncio.Future] = {} + + def _subscribe(session: str) -> "asyncio.Queue[dict]": + q: "asyncio.Queue[dict]" = asyncio.Queue(maxsize=_QUEUE_MAX) + subscribers.setdefault(session, []).append(q) + # Replay anything enqueued while this session had no live subscriber. + for cmd in backlog.pop(session, []) or []: + try: + q.put_nowait(cmd) + except asyncio.QueueFull: + break + return q + + def _unsubscribe(session: str, q: "asyncio.Queue[dict]") -> None: + subs = subscribers.get(session) + if subs and q in subs: + subs.remove(q) + if not subs: + subscribers.pop(session, None) + + def _broadcast(session: str, cmd: dict) -> int: + subs = subscribers.get(session) or [] + if subs: + for q in list(subs): + try: + q.put_nowait(cmd) + except asyncio.QueueFull: + pass + return len(subs) + # No live subscriber yet: hold it so a subscriber that connects shortly + # after still receives the command (bounded ring buffer). + bl = backlog.setdefault(session, []) + bl.append(cmd) + if len(bl) > _QUEUE_MAX: + del bl[0: len(bl) - _QUEUE_MAX] + return 0 + + def _scoped(session: Optional[str], owner: Optional[str]) -> str: + return f"{owner or 'anon'}:{session or 'default'}" + + @router.post("/api/editor/plan") + async def plan(request: Request) -> JSONResponse: + owner = _user(request) + try: + body = await request.json() + except Exception: + body = {} + prompt = str(body.get("prompt") or "").strip() + state = body.get("state") or {} + mode = str(body.get("mode") or "plan").strip().lower() + screenshot = body.get("screenshot") + agent_model = str(body.get("agent_model") or "").strip() + if not prompt and mode != "check": + return JSONResponse({"reply": "No prompt provided.", "planner": "none", "steps": [], + "agent_model": agent_model or editor_agent_model.local_model()}) + + system = _CHECK_SYSTEM_PROMPT if mode == "check" else _SYSTEM_PROMPT + text = json.dumps({"prompt": prompt, "state": state}, ensure_ascii=False) + content: Any = text + if isinstance(screenshot, str) and screenshot.startswith("data:image/"): + content = [ + {"type": "text", "text": text}, + {"type": "image_url", "image_url": {"url": screenshot}}, + ] + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": content}, + ] + + raw: Any = None + planner = _EDITOR_PLANNER + editor_agent_model.set_override(agent_model) + editor_agent_model.set_last_model("", "") + if planner is not None: + try: + raw = planner(messages, owner) + if inspect.isawaitable(raw): + raw = await raw + except Exception: + raw = None + _used = editor_agent_model.get_last_model() + model_used = (_used[0] if _used and _used[0] + else (agent_model or editor_agent_model.local_model())) + parsed = _parse_plan(raw) + if isinstance(parsed, dict): + parsed["agent_model"] = model_used + if mode != "check": + _special = _plan_special(prompt, state, _image_provider_available(owner)) + if _special: + _special_steps, _special_reply, _ = _special + return JSONResponse({ + "reply": _special_reply, + "planner": "deterministic", + "steps": style_guide.enforce_guards(_special_steps) or _special_steps, + "agent_model": model_used, + }) + if parsed and mode != "check": + _strengthened = style_guide.strengthen(prompt, list(parsed.get("steps") or [])) + if _strengthened != parsed.get("steps"): + parsed["steps"] = _strengthened + parsed["planner"] = str(parsed.get("planner") or "llm") + "+guide" + # Keep /plan and /agent consistent: a generative intent must never be + # left on a deterministic substitute (e.g. a model returning only + # selection/adjustment steps for 'remove the person on the left'). + _generative = _ensure_generative(prompt, list(parsed.get("steps") or [])) + if _generative != parsed.get("steps"): + parsed["steps"] = _generative + parsed["planner"] = str(parsed.get("planner") or "llm") + "+generative" + # Same safety guards the /agent path applies, so /plan never returns + # a stacked-lift or a global adjustment after a masked recolour. + parsed["steps"] = style_guide.enforce_guards(list(parsed.get("steps") or [])) + # A bare whole-image tone plan for an unspecified prompt is a guess: + # ask the user instead of silently tweaking. + _maybe = _unsure_plan(prompt, parsed) + if _maybe is not parsed and isinstance(_maybe, dict): + _maybe["agent_model"] = model_used + parsed = _maybe + + if mode == "check": + # Never let a self-check re-plan substitute a generative placeImage + # for a deterministic recolour; re-issue the masked-hue plan instead. + _recolor = style_guide.recolor_plan(prompt) + if _recolor and _plan_uses_ai_content(list((parsed or {}).get("steps") or [])): + return JSONResponse({ + "reply": str(_recolor["reply"]), + "planner": "deterministic", + "steps": style_guide.enforce_guards(list(_recolor["steps"])), + "agent_model": model_used, + }) + if parsed and (parsed.get("done") or parsed.get("steps")): + return JSONResponse(parsed) + return JSONResponse({ + "done": True, + "reply": (parsed or {}).get("reply") or "The edit looks complete.", + "planner": (parsed or {}).get("planner") or "check", + "steps": [], + "agent_model": model_used, + }) + if parsed: + return JSONResponse(parsed) + _fallback = _fallback_plan(prompt) + _fallback["agent_model"] = model_used + return JSONResponse(_fallback) + + @router.post("/api/editor/bridge/command") + async def bridge_command(request: Request) -> JSONResponse: + owner = _user(request) + try: + body = await request.json() + except Exception: + body = {} + session = _scoped(body.get("session"), owner) + steps = _normalize_steps(body) + wait = bool(body.get("wait")) + timeout = float(body.get("timeout") or _BRIDGE_DEFAULT_TIMEOUT) + ids: List[str] = [] + wait_ids: List[str] = [] + for step in steps: + cid = uuid.uuid4().hex + ids.append(cid) + if wait: + pending[cid] = asyncio.get_event_loop().create_future() + wait_ids.append(cid) + _broadcast(session, {"id": cid, "op": step["op"], "params": step["params"]}) + if wait and len(wait_ids) == 1: + try: + result = await asyncio.wait_for(pending[wait_ids[0]], timeout=timeout) + return JSONResponse(result) + except asyncio.TimeoutError: + pending.pop(wait_ids[0], None) + return JSONResponse({"ok": False, "error": "timeout", "id": wait_ids[0]}) + return JSONResponse({"queued": ids, "session": session}) + + @router.get("/api/editor/bridge/events") + async def bridge_events(request: Request, session: str = "default") -> StreamingResponse: + owner = _user(request) + scoped = _scoped(session, owner) + + async def gen(): + q = _subscribe(scoped) + try: + yield ": connected\n\n" + while not _shutdown.is_shutting_down(): + try: + cmd = await asyncio.wait_for(q.get(), timeout=15) + yield f"data: {json.dumps(cmd)}\n\n" + except asyncio.TimeoutError: + yield ": keepalive\n\n" + finally: + _unsubscribe(scoped, q) + + return StreamingResponse(gen(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + @router.post("/api/editor/bridge/result") + async def bridge_result(request: Request) -> JSONResponse: + try: + body = await request.json() + except Exception: + body = {} + cid = str(body.get("id") or "") + fut = pending.pop(cid, None) + if fut is not None and not fut.done(): + fut.set_result({ + "id": cid, "ok": bool(body.get("ok")), + "result": body.get("result"), "error": body.get("error"), + "screenshot": body.get("screenshot"), + }) + return JSONResponse({"ok": True}) + + @router.post("/api/editor/bridge/await") + async def bridge_await(request: Request) -> JSONResponse: + try: + body = await request.json() + except Exception: + body = {} + cid = str(body.get("id") or "") + timeout = float(body.get("timeout") or _BRIDGE_DEFAULT_TIMEOUT) + fut = pending.get(cid) + if fut is None: + fut = asyncio.get_event_loop().create_future() + pending[cid] = fut + try: + result = await asyncio.wait_for(fut, timeout=timeout) + return JSONResponse(result) + except asyncio.TimeoutError: + pending.pop(cid, None) + return JSONResponse({"ok": False, "error": "timeout", "id": cid}) + + _AGENT_SYSTEM_PROMPT = ( + "You are the photo editor agent. You drive the open image editor and " + "may also run AI image operations. Available editor ops: " + + ", ".join(OP_CATALOG) + + ", plus ai_edit_image and generate_image. " + "Respond with ONLY a JSON object: {\"reply\": , " + "\"steps\": [{\"op\": , \"params\": {...}}]}. " + + _ROUTING_GUIDANCE + " " + + _CANONICAL_GUIDANCE + " " + _STYLE_GUIDANCE + " " + "Prefer AI content ops for generative requests: \"remove the person on the left\" " + "-> op \"ai_edit_image\" op \"inpaint\" with a prompt; \"add a full moon in the sky\" " + "-> op \"generate_image\" with a prompt. " + "Use deterministic editor ops (adjustments.apply, filters.apply, selection.*, " + "layers.placeImage, text.add, ...) only for non-generative, precise edits. " + "Keep steps minimal and ordered. " + + _SCHEMA_NOTE + ) + + def _scoped_editor_tools() -> List[str]: + """The exact tool list offered to the model for a photo_editor turn.""" + fallback = [ + "plan_edit", "run_editor_command", "get_editor_screenshot", + "ai_edit_image", "generate_image", "ask_user", + ] + try: + from src.tool_policy import ToolPolicy + from src.tool_schemas import FUNCTION_TOOL_SCHEMAS + from src.turn_contract import resolve_turn_contract + + contract = resolve_turn_contract( + capabilities={"photo_editor"}, + schemas=FUNCTION_TOOL_SCHEMAS, + policy=ToolPolicy(), + required_capabilities={"photo_editor"}, + ) + return sorted(contract.offered) or fallback + except Exception: + return fallback + + @router.post("/api/editor/agent") + async def editor_agent(request: Request) -> JSONResponse: + """Scoped photo_editor agent turn (Quick Edit). + + Offers ONLY the editor tool family plus ask_user, plans edits, and + drives the open editor through the owner-scoped bridge. AI edit and + generation actions land as new layers; the original is never + overwritten. + """ + owner = _user(request) + try: + body = await request.json() + except Exception: + body = {} + prompt = str(body.get("prompt") or "").strip() + session = str(body.get("editor_session") or body.get("session") or "default") + state = body.get("state") if isinstance(body.get("state"), dict) else {} + screenshot = body.get("screenshot") + # Optional user-picked image model (from the editor model selector). + model_choice = str(body.get("model") or body.get("image_model") or "").strip() + # Optional user-picked PLANNER model (the editor agent/planner model). + agent_model = str(body.get("agent_model") or "").strip() + tools_offered = _scoped_editor_tools() + scoped = _scoped(session, owner) + trace: List[Dict[str, Any]] = [ + {"type": "session", "session": scoped, "tools_offered": tools_offered}, + ] + if not prompt: + return JSONResponse({ + "ok": False, "reply": "No prompt provided.", + "tools_offered": tools_offered, "steps": [], "trace": trace, + "agent_model": agent_model or editor_agent_model.local_model(), + }) + text = json.dumps({"prompt": prompt, "state": state}, ensure_ascii=False) + content: Any = text + if isinstance(screenshot, str) and screenshot.startswith("data:image/"): + content = [ + {"type": "text", "text": text}, + {"type": "image_url", "image_url": {"url": screenshot}}, + ] + messages = [ + {"role": "system", "content": _AGENT_SYSTEM_PROMPT}, + {"role": "user", "content": content}, + ] + raw = None + planner = _EDITOR_PLANNER + editor_agent_model.set_override(agent_model) + editor_agent_model.set_last_model("", "") + if planner is not None: + try: + raw = planner(messages, owner) + if hasattr(raw, "__await__"): + raw = await raw + except Exception: + raw = None + _used = editor_agent_model.get_last_model() + model_used = (_used[0] if _used and _used[0] + else (agent_model or editor_agent_model.local_model())) + plan = _parse_plan(raw) or _fallback_plan(prompt) + _strengthened = style_guide.strengthen(prompt, list(plan.get("steps") or [])) + if _strengthened != plan.get("steps"): + plan = {**plan, "steps": _strengthened, + "planner": f"{plan.get('planner') or 'llm'}+guide"} + kind = _generative_kind(prompt) + provider_available = _image_provider_available(owner) + # Deterministic overrides for intents the model reliably botches: canvas + # format/crop/extend, text + layer styles, and PSD export. + _special = _plan_special(prompt, state, provider_available) + special_note = "" + if _special: + special_steps, special_reply, special_note = _special + steps = style_guide.enforce_guards(special_steps) or list(special_steps) + reply = special_reply + kind = None + plan = {**plan, "steps": steps, + "planner": f"{plan.get('planner') or 'llm'}+deterministic"} + else: + steps = _ensure_generative(prompt, list(plan.get("steps") or [])) + steps = style_guide.enforce_guards(steps) + if steps != (plan.get("steps") or []): + plan = {**plan, "steps": steps, + "planner": f"{plan.get('planner') or 'llm'}+generative"} + reply = str(plan.get("reply") or "") + # A bare whole-image tone plan for an unspecified prompt is a guess: + # ask the user instead of silently tweaking. + # A bare whole-image tone plan for an unspecified prompt is a guess: + # ask the user instead of silently tweaking. Only replace when the + # guard actually changed the plan (identity), so a model-supplied + # ask_user never wipes out a concrete generative edit. + _base_plan = {**plan, "steps": steps} + _maybe = _unsure_plan(prompt, _base_plan) + if _maybe is not _base_plan and isinstance(_maybe, dict) and _maybe.get("ask_user"): + plan = _maybe + steps = [] + reply = str(_maybe.get("reply") or "") + trace.append({ + "type": "plan", "tool": "plan_edit", + "planner": plan.get("planner"), "steps": steps, + "image_provider": provider_available, "agent_model": model_used, + }) + # A vague prompt with no concrete defect asks the user instead of + # guessing a token edit (2-3 concrete options). + ask_user_payload = plan.get("ask_user") + if isinstance(ask_user_payload, dict) and not steps and not _special: + trace.append({"type": "ask_user", + "question": ask_user_payload.get("question")}) + return JSONResponse({ + "ok": True, + "ask_user": ask_user_payload, + "ask_user_pending": True, + "reply": reply or str(ask_user_payload.get("question") or ""), + "tools_offered": tools_offered, + "steps": [], + "trace": trace, + "results": [], + "agent_model": model_used, + }) + # No image provider: stop before any edit runs and tell the user plainly + # (zero canvas changes; nothing to roll back). + if (kind or _plan_uses_ai_content(steps)) and not provider_available: + trace.append({"type": "blocked", "reason": "image_provider_unavailable"}) + return JSONResponse({ + "ok": False, + "blocked": True, + "needs_provider": True, + "reply": _PROVIDER_REQUIRED_MESSAGE, + "message": _PROVIDER_REQUIRED_MESSAGE, + "image_provider": False, + "tools_offered": tools_offered, + "steps": steps, + "trace": trace, + "results": [], + "agent_model": model_used, + }) + results: List[Dict[str, Any]] = [] + for step in steps: + op = step.get("op") + params = step.get("params") or {} + if op in ("ai_edit_image", "generate_image"): + try: + from src.agent_tools.editor_tools import AiEditImageTool, GenerateImageLayerTool + except Exception: + results.append({"op": op, "ok": False, "error": "editor tools unavailable"}) + break + tool_payload = dict(params) + tool_payload.setdefault("session", session) + # Route to the user-picked model when one is selected. + if model_choice: + tool_payload.setdefault("model", model_choice) + # Hand the AI step the canvas image the client already captured + # for this request. Without it the tool falls back to a separate + # bridge screenshot, which is not reliably answered right after a + # selection step (the masked path was losing the canvas image). + if isinstance(screenshot, str) and screenshot.startswith("data:image/"): + tool_payload.setdefault("image", screenshot) + tool_cls = AiEditImageTool if op == "ai_edit_image" else GenerateImageLayerTool + try: + _desc, res = await tool_cls().execute( + json.dumps(tool_payload), + {"owner": owner, "session_id": scoped}, + ) + except Exception as exc: + res = {"error": str(exc), "exit_code": 1} + res = res if isinstance(res, dict) else {} + ok = res.get("exit_code", 0) == 0 + step_result = { + k: res.get(k) for k in ("image_id", "image_url", "placed", "output", "model") + if k in res + } + results.append({ + "op": op, "ok": ok, "result": step_result, + # Surface the real provider/tool failure so the reply can + # say what went wrong instead of a bare ok:true/ok:false. + "error": None if ok else (res.get("error") or res.get("output") or "it did not complete"), + }) + trace.append({"type": "step", "op": op, "ok": ok, "result": step_result}) + if not ok: + break + continue + cid = uuid.uuid4().hex + pending[cid] = asyncio.get_event_loop().create_future() + _broadcast(scoped, {"id": cid, "op": op, "params": params}) + try: + res = await asyncio.wait_for(pending[cid], timeout=_BRIDGE_DEFAULT_TIMEOUT) + except asyncio.TimeoutError: + pending.pop(cid, None) + results.append({"op": op, "ok": False, "error": "timeout"}) + trace.append({"type": "step", "op": op, "ok": False, "error": "timeout"}) + break + results.append(res) + trace.append({"type": "step", "op": op, "ok": res.get("ok"), "result": res.get("result")}) + if res.get("ok") is False: + break + # Post-run verification: read the ACTUAL canvas back and base the reply + # on executed results + final document.info, never the plan. + final_state: Dict[str, Any] = {} + if steps and not any(r.get("ok") is False for r in results): + try: + _info_id = uuid.uuid4().hex + pending[_info_id] = asyncio.get_event_loop().create_future() + _broadcast(scoped, {"id": _info_id, "op": "document.info", "params": {}}) + _info = await asyncio.wait_for(pending[_info_id], timeout=60) + if isinstance(_info, dict) and isinstance(_info.get("result"), dict): + final_state = _info["result"] + except Exception: + final_state = {} + planned_reply = reply + reply = _verified_reply(prompt, steps, results, final_state, planned_reply) + if special_note and special_note.strip() not in reply: + reply = (reply + " " + special_note.strip()).strip() + return JSONResponse({ + "ok": not any(r.get("ok") is False for r in results), + "reply": reply or "Done.", + "planned_reply": planned_reply, + "image_provider": provider_available, + "final_state": final_state, + "tools_offered": tools_offered, + "tools": ["plan_edit"], + "steps": steps, + "trace": trace, + "results": results, + "agent_model": model_used, + }) + + @router.get("/api/editor/style-guide") + async def style_guide_route() -> JSONResponse: + """The shared intent -> ops guide (single source of truth). + + The browser fallback planner fetches this so the client and server + routing rules cannot drift. + """ + return JSONResponse(style_guide.as_json()) + + @router.get("/api/editor/image-models") + async def editor_image_models(request: Request) -> JSONResponse: + """List the image models available from configured, owner-visible providers. + + Each entry is ``{id, label, kind, provider}`` where ``id`` is the model id + to pass as ``model`` to ai_edit_image / generate_image, and ``kind`` is + ``edit`` (editing/inpainting models) or ``generate`` (text-to-image). + """ + owner = _user(request) + models: List[Dict[str, Any]] = [] + seen: set = set() + try: + from core.database import SessionLocal, ModelEndpoint + from src.auth_helpers import owner_filter + from src.endpoint_resolver import _endpoint_enabled_models + from src.image_model_ids import looks_like_image_generation_model + + db = SessionLocal() + try: + query = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), # noqa: E712 + ModelEndpoint, + owner, + ) + for ep in query.all(): + # Image-only endpoints (the local Qwen server) list their + # models verbatim. Gateway/chat endpoints (OpenRouter, …) + # mix chat and image models, so expose only ids that are + # recognisably image-capable — the same test the request + # router uses. + image_endpoint = getattr(ep, "model_type", None) == "image" + provider = getattr(ep, "name", None) or getattr(ep, "base_url", "") or "image" + for name in _endpoint_enabled_models(ep): + if not image_endpoint and not looks_like_image_generation_model(name): + continue + key = (provider, name) + if key in seen: + continue + seen.add(key) + models.append({ + "id": name, + "label": name, + "kind": "edit" if _EDIT_MODEL_RE.search(name) else "generate", + "provider": provider, + "endpoint_id": getattr(ep, "id", None), + "base_url": getattr(ep, "base_url", "") or "", + }) + finally: + db.close() + except Exception: + logger.warning("editor image-models lookup failed", exc_info=True) + return JSONResponse({"models": models}) + + @router.get("/api/editor/agent-models") + async def editor_agent_models(request: Request) -> JSONResponse: + """List the models the editor agent/planner can run on. + + Each entry is ``{id, label, source, vision, base_url}`` where ``id`` is + passed as ``agent_model`` to /api/editor/plan or /api/editor/agent, and + ``source`` is ``local`` (the local Qwen3.5-9B vision server) or + ``hosted`` (the configured default/utility chat model that the planner + falls back to). + """ + owner = _user(request) + try: + models = await editor_agent_model.list_editor_agent_models(owner) + except Exception: + logger.warning("editor agent-models lookup failed", exc_info=True) + models = [] + return JSONResponse({ + "models": models, + "probe": editor_agent_model.probe_status(), + }) + + @router.post("/api/editor/self-check") + async def editor_self_check(request: Request) -> JSONResponse: + """Same-origin VL self-check: compare BEFORE/AFTER against the intent. + + The local VL server (seneca :8097) sends no CORS headers, so the browser + cannot call it directly and the self-check silently "passed". This route + proxies the two images + intent to the agent VLM and returns + ``{done, issues, retry_hint, text}`` (``done=True`` with + ``reason="unavailable"`` on any failure so the client never loops). + """ + owner = _user(request) + try: + body = await request.json() + except Exception: + body = {} + result = await editor_agent_model.self_check( + body.get("before"), + body.get("after"), + str(body.get("prompt") or body.get("intent") or ""), + model=str(body.get("model") or "").strip() or None, + owner=owner, + ) + return JSONResponse(result) + + return router diff --git a/routes/email/email_routes.py b/routes/email/email_routes.py index b8329eef2..294585e79 100644 --- a/routes/email/email_routes.py +++ b/routes/email/email_routes.py @@ -43,6 +43,7 @@ from starlette.concurrency import run_in_threadpool from fastapi import APIRouter, Query, UploadFile, File, BackgroundTasks, HTTPException, Depends, Request from fastapi.responses import FileResponse, StreamingResponse from src.constants import DATA_DIR +from src import shutdown as _shutdown from src.path_confinement import confine from src.llm_core import llm_call_async @@ -1834,13 +1835,55 @@ def setup_email_routes(): return None return v[1] + _LIST_STALE = {} + + def _list_stale_get(key): + return _LIST_STALE.get(key) + + def _list_stale_put(key, value): + _LIST_STALE[key] = value + if len(_LIST_STALE) > 96: + for k in list(_LIST_STALE.keys())[:-48]: + _LIST_STALE.pop(k, None) + + _LIST_REFRESHING = set() + def _list_cache_put(key, value): _LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value) + # Also snapshot the last-good response for stale-while-revalidate. + _list_stale_put(key, value) # Cap size if len(_LIST_CACHE) > 64: for k in list(_LIST_CACHE.keys())[:-32]: _LIST_CACHE.pop(k, None) + def _schedule_list_refresh(sync_args, cache_key, folder, account_id, owner): + """Refresh a list snapshot off the event loop, at most once per key. + + Runs the blocking IMAP fetch in a daemon thread, then repopulates both + the TTL cache and the stale snapshot so the next list view is warm. + """ + if cache_key in _LIST_REFRESHING: + return + _LIST_REFRESHING.add(cache_key) + + def _run(): + try: + refreshed = _list_emails_sync(*sync_args) + if refreshed and not refreshed.get("error"): + _list_cache_put(cache_key, refreshed) + try: + _schedule_recent_email_warm( + refreshed.get("emails") or [], folder, account_id, owner) + except Exception: + pass + except Exception as exc: # pragma: no cover - background best effort + logger.debug("email list background refresh failed: %s", exc) + finally: + _LIST_REFRESHING.discard(cache_key) + + _threading.Thread(target=_run, daemon=True, name="email-list-refresh").start() + def _folder_cache_get(account_id, owner): key = (account_id or "", owner or "") v = _FOLDER_CACHE.get(key) @@ -2867,6 +2910,46 @@ def setup_email_routes(): owner, account_id or "", folder, filter, limit, offset, elapsed_ms, ) return cached + # Stale-while-revalidate: never block a plain list view on IMAP while + # we already hold a recent snapshot or a durable index. Serve it now + # with a `stale` marker and refresh in the background so the next view + # is warm. Blocking IMAP only remains for a genuine cold start with no + # snapshot and no indexed rows (e.g. a brand-new account/folder). + if not from_addr and not has_attachments and not (date_from or date_to): + snapshot = _list_stale_get(ck) + if snapshot is None: + try: + indexed_emails, indexed_total, indexed_at = _email_index_list( + owner, account_id, folder, filter, limit, offset, False, + ) + except Exception: + indexed_emails, indexed_total, indexed_at = [], 0, None + if indexed_total: + snapshot = { + "emails": indexed_emails, + "total": indexed_total, + "folder": folder, + "offset": offset, + "sync": { + "source": "index", + "indexed": indexed_total, + "updated_at": indexed_at, + }, + } + if snapshot is not None: + payload = dict(snapshot) + payload["emails"] = list(payload.get("emails") or []) + _hide_unlinked_calendar_tags(payload["emails"]) + sync_meta = dict(payload.get("sync") or {}) + sync_meta["source"] = "cache_stale" if _list_stale_get(ck) is not None else "index" + sync_meta["stale"] = True + payload["sync"] = sync_meta + _schedule_list_refresh( + (folder, limit, offset, filter, account_id, from_addr, + bool(has_attachments), owner, True, date_from or "", date_to or ""), + ck, folder, account_id, owner, + ) + return payload result = await _asyncio.to_thread( _list_emails_sync, folder, limit, offset, filter, account_id, from_addr, bool(has_attachments), owner, manual_refresh, date_from or "", date_to or "", @@ -6717,7 +6800,7 @@ def setup_email_routes(): await queue.put({'type': 'result', **result}) task = asyncio.create_task(run()) try: - while True: + while not _shutdown.is_shutting_down(): item = await queue.get() yield 'data: ' + json.dumps(item) + '\n\n' if item.get('type') == 'result': diff --git a/routes/gallery/gallery_routes.py b/routes/gallery/gallery_routes.py index 276ac90a5..c856223c3 100644 --- a/routes/gallery/gallery_routes.py +++ b/routes/gallery/gallery_routes.py @@ -1,14 +1,17 @@ """Gallery routes — browsable library for photos and AI-generated images.""" import os +import asyncio import base64 +import gc import hashlib import io import logging import re +import time import uuid from pathlib import Path -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, Tuple from fastapi import APIRouter, HTTPException, Query, Request @@ -34,6 +37,241 @@ _SAM_STATE: Dict[str, Any] = {} _GROUNDING_STATE: Dict[str, Any] = {} +def _rss_mb() -> float: + """Current process RSS in MB (from /proc; -1.0 when unavailable).""" + try: + with open("/proc/self/status", "r") as fh: + for line in fh: + if line.startswith("VmRSS:"): + return round(int(line.split()[1]) / 1024.0, 1) + except Exception: + pass + return -1.0 + + +def _image_op_log_rss(op: str, phase: str) -> None: + logger.info("image-op %s %s rss_mb=%.1f", op, phase, _rss_mb()) + + +def _local_image_models_enabled() -> bool: + """Whether GB-scale local image models may be loaded IN-PROCESS. + + Default OFF: loading rembg/SAM/OWL-ViT/Real-ESRGAN in the web process + routinely OOM-kills memory-constrained hosts. Enable with + ``ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS=1`` or point ``ODYSSEUS_IMAGE_OPS_BASE`` + at a remote service that implements the same ``/api/image/*`` endpoints. + """ + return os.getenv("ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS", "").strip().lower() in ( + "1", "true", "yes", "on", + ) + + +def _remote_image_ops_base() -> str: + return (os.getenv("ODYSSEUS_IMAGE_OPS_BASE", "") or "").strip().rstrip("/") + + +def _outpaint_timeout() -> float: + """Provider timeout for the outpaint/extend proxy. + + Tall extensions can take up to ~330 s (the provider's own timeout is + 900 s), so the proxy waits 900 s by default — otherwise a long extend is + cut off at the proxy. Override with ``ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT``. + """ + try: + return float(os.getenv("ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT", "900") or 900) + except (TypeError, ValueError): + return 900.0 + + +def _unload_image_models() -> None: + """Drop lazily-loaded torch models so their RAM returns to the OS.""" + if not _SAM_STATE and not _GROUNDING_STATE: + return + _SAM_STATE.clear() + _GROUNDING_STATE.clear() + try: + import torch # type: ignore + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + gc.collect() + _image_op_log_rss("models", "unloaded") + + +async def _route_heavy_image_op(path: str, body: dict, *, label: str): + """Proxy a heavy image op to the remote ops service when configured, else + run it in-process only when explicitly enabled. + + Returns the remote JSON on success, or ``None`` to signal the caller should + run it in-process; raises HTTPException(503) with a clear message otherwise + so the app never loads a GB-scale model on a memory-constrained host. + """ + base = _remote_image_ops_base() + if base: + _image_op_log_rss(label, f"proxy->{base}") + try: + import httpx + + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post(base + path, json=body) + if resp.status_code == 200: + return resp.json() + logger.warning("image-op %s remote %s -> %s", label, path, resp.status_code) + raise HTTPException( + 503, + f"{label} is temporarily unavailable (remote service returned {resp.status_code}).", + ) + except HTTPException: + raise + except Exception as exc: + logger.warning("image-op %s remote %s failed: %s", label, path, exc) + raise HTTPException( + 503, + f"{label} is temporarily unavailable (remote service unreachable).", + ) from exc + if _local_image_models_enabled(): + return None + raise HTTPException(503, f"{label} is temporarily unavailable on this server.") + + +def _xywh_to_xyxy(bbox): + """Convert a remote image-ops [x, y, w, h] bbox to this app's [x1,y1,x2,y2].""" + try: + x, y, w, h = (float(v) for v in bbox) + return [int(x), int(y), int(x + w), int(y + h)] + except Exception: + return bbox + + +async def _remote_remove_background(body: dict): + """Proxy background removal to the remote ops service (seneca contract). + + Endpoint: POST {base}/v1/images/remove-background (multipart: image, + optional hint_mask, model, return_mask). Returns {"image": } + to match this route's local response shape, or None when the caller should + run in-process (local models explicitly enabled); raises 503 otherwise. + """ + base = _remote_image_ops_base() + if not base: + if _local_image_models_enabled(): + return None + raise HTTPException(503, "Background removal is temporarily unavailable on this server.") + img_bytes = _b64_bytes(body.get("image")) + hint_bytes = None + if body.get("hint_mask"): + hint_bytes = _b64_bytes(body.get("hint_mask"), label="hint_mask") + _image_op_log_rss("remove-bg", f"proxy->{base}") + try: + import httpx + + files = {"image": ("image.png", img_bytes, "image/png")} + if hint_bytes: + files["hint_mask"] = ("hint.png", hint_bytes, "image/png") + data = {"model": str(body.get("model") or ""), "return_mask": "true"} + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post(base + "/v1/images/remove-background", files=files, data=data) + if resp.status_code == 200: + payload = resp.json() + items = payload.get("data") or [] + if not items or not items[0].get("b64_json"): + raise HTTPException(503, "Background removal is temporarily unavailable (remote returned no image).") + out = {"image": items[0]["b64_json"]} + if items[0].get("mask_b64"): + out["mask"] = items[0]["mask_b64"] + if items[0].get("bbox"): + out["bbox"] = items[0]["bbox"] + return out + raise HTTPException( + 503, f"Background removal is temporarily unavailable (remote service returned {resp.status_code})." + ) + except HTTPException: + raise + except Exception as exc: + logger.warning("image-op remove-bg remote failed: %s", exc) + raise HTTPException(503, "Background removal is temporarily unavailable (remote service unreachable).") from exc + + +async def _remote_smart_mask(body: dict): + """Proxy SAM/grounding masking to the remote ops service (seneca contract). + + Endpoint: POST {base}/v1/images/mask (multipart: image + one of prompt/box/ + points). Returns {"mask": , "bbox": [...], "model", "device"} to + match this route's local response shape, or None when the caller should run + in-process; raises 503 otherwise. + """ + base = _remote_image_ops_base() + if not base: + if _local_image_models_enabled(): + return None + raise HTTPException(503, "Smart mask is temporarily unavailable on this server.") + img_bytes = _b64_bytes(body.get("image")) + points = body.get("points") or [] + box = body.get("box") + text = (body.get("text") or body.get("query") or "").strip() + if not points and not box and not text: + raise HTTPException(400, "Provide at least one point, box, or object text") + _image_op_log_rss("mask", f"proxy->{base}") + try: + import httpx + + data: Dict[str, Any] = {} + if text: + data["prompt"] = text + if box: + try: + x1, y1, x2, y2 = (float(v) for v in box) + data["box"] = ",".join(str(int(v)) for v in (x1, y1, x2 - x1, y2 - y1)) + except Exception as exc: + raise HTTPException(400, "Box must be [x1, y1, x2, y2]") from exc + if points: + import json as _json + + data["points"] = _json.dumps( + [[float(p["x"]), float(p["y"]), int(p.get("label", 1))] for p in points] + ) + if body.get("multimask") is not None: + data["multimask"] = str(bool(body.get("multimask"))).lower() + if body.get("dilate_px") is not None: + data["dilate_px"] = str(int(body.get("dilate_px"))) + files = {"image": ("image.png", img_bytes, "image/png")} + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post(base + "/v1/images/mask", files=files, data=data) + if resp.status_code == 200: + payload = resp.json() + masks = payload.get("masks") or [] + candidates = [m for m in masks if m.get("b64_json")] + if not candidates: + # Empty selection is a normal 200 outcome (never a 400): the + # editor shows a "nothing selected" message and changes nothing. + return { + "mask": None, + "bbox": None, + "masks": [], + "model": payload.get("model") or "sam+grounding", + "device": payload.get("device", ""), + "error": "Nothing selected for that selection.", + } + best = max(candidates, key=lambda m: float(m.get("score") or 0.0)) + return { + "mask": best["b64_json"], + "bbox": _xywh_to_xyxy(best.get("bbox")), + "label": best.get("label"), + "masks": masks, + "model": payload.get("model") or "sam+grounding", + "device": payload.get("device", ""), + } + raise HTTPException( + 503, f"Smart mask is temporarily unavailable (remote service returned {resp.status_code})." + ) + except HTTPException: + raise + except Exception as exc: + logger.warning("image-op mask remote failed: %s", exc) + raise HTTPException(503, "Smart mask is temporarily unavailable (remote service unreachable).") from exc + + def _b64_to_pil_image(image_b64: str, *, mode: str = "RGBA"): if not image_b64: raise HTTPException(400, "Missing image") @@ -50,12 +288,79 @@ def _b64_to_pil_image(image_b64: str, *, mode: str = "RGBA"): raise HTTPException(400, "Invalid image") from exc +def _b64_bytes(value: str, *, label: str = "image") -> bytes: + """Decode raw base64 OR a ``data:image/...;base64,`` URL to bytes. + + The image-edit tools post data URLs, while the browser posts raw base64, + so every decode site must accept both. + """ + if not value: + raise HTTPException(400, f"Missing {label}") + text = str(value) + if "," in text and text.split(",", 1)[0].startswith("data:"): + text = text.split(",", 1)[1] + try: + return base64.b64decode(text) + except Exception as exc: + raise HTTPException(400, f"Invalid {label}") from exc + + def _pil_image_to_b64(img, *, fmt: str = "PNG") -> str: buf = io.BytesIO() img.save(buf, format=fmt) return base64.b64encode(buf.getvalue()).decode("ascii") +def _prepare_local_edit(src_b64, mask_b64, max_side): + """Decode/normalize/cap an edit's inputs to PNG bytes at a model-friendly size. + + CPU-bound (a 4000x3000 decode+resize is ~0.2 s); run it via a thread off the + event loop. Returns (source_png_bytes, mask_png_bytes, model_w, model_h, orig_size). + """ + from PIL import Image + + img_bytes = _b64_bytes(src_b64) + source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA") + if mask_b64: + mask_bytes = _b64_bytes(mask_b64, label="mask") + mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L") + else: + mask_png = Image.new("L", source_png.size, 255) + orig_size = source_png.size + if max(orig_size) > max_side: + scale = float(max_side) / max(orig_size) + model_w = max(16, int(round(orig_size[0] * scale / 16.0)) * 16) + model_h = max(16, int(round(orig_size[1] * scale / 16.0)) * 16) + else: + model_w, model_h = orig_size + if (model_w, model_h) != orig_size: + source_work = source_png.resize((model_w, model_h), Image.LANCZOS) + mask_work = mask_png.resize((model_w, model_h), Image.LANCZOS) + else: + source_work, mask_work = source_png, mask_png + src_buf = io.BytesIO() + source_work.save(src_buf, format="PNG") + mask_buf = io.BytesIO() + mask_work.save(mask_buf, format="PNG") + return src_buf.getvalue(), mask_buf.getvalue(), model_w, model_h, orig_size + + +def _restore_local_b64(raw_b64, orig_size, model_size): + """Resize a provider result b64 back to the original dimensions (CPU-bound).""" + if tuple(model_size) == tuple(orig_size): + return raw_b64 + from PIL import Image + + try: + im = Image.open(io.BytesIO(base64.b64decode(raw_b64))).convert("RGBA") + im = im.resize(tuple(orig_size), Image.LANCZOS) + ob = io.BytesIO() + im.save(ob, format="PNG") + return base64.b64encode(ob.getvalue()).decode("ascii") + except Exception: + return raw_b64 + + def _load_sam_backend(): model_id = os.getenv("ODYSSEUS_SAM_MODEL", "facebook/sam-vit-base") cached = _SAM_STATE.get(model_id) @@ -301,7 +606,61 @@ def _first_visible_image_endpoint(db, owner: str | None): for ep in endpoints: if getattr(ep, "owner", None) == owner: return ep - return endpoints[0] if endpoints else None + if endpoints: + return endpoints[0] + # Fallback: the image model may live on a *mixed* endpoint (model_type != + # "image"), e.g. qwen-image-edit / gpt-image-1 on a shared OpenAI-compatible + # gateway or a diffusion server registered as "chat". Chat image generation + # already resolves its model by name across every enabled endpoint, so the + # editor must find the same model here instead of reporting "no endpoint". + return _first_mixed_image_endpoint(db, owner) + + +def _first_mixed_image_endpoint(db, owner: str | None): + """Best-effort image endpoint when no image-only endpoint is registered. + + Returns an enabled, owner-visible endpoint whose configured model list + includes an image-generation model, preferring the admin's configured + ``image_model`` when it is one of them. + """ + import json + from src.auth_helpers import owner_filter + from src.image_model_ids import looks_like_image_generation_model + + preferred = "" + try: + from src.settings import load_settings + preferred = str((load_settings() or {}).get("image_model") or "").strip() + except Exception: + preferred = "" + + query = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), # noqa: E712 + ModelEndpoint, + owner, + ) + fallback = None + for ep in query.all(): + if getattr(ep, "model_type", None) == "image": + continue + models = set() + for field in ("cached_models", "pinned_models"): + try: + values = getattr(ep, field, None) or [] + values = json.loads(values) if isinstance(values, str) else values + models.update(v for v in values if isinstance(v, str)) + except (TypeError, ValueError): + pass + matches = {m for m in models if looks_like_image_generation_model(m)} + if not matches: + continue + if preferred and preferred in matches: + return ep + if owner and getattr(ep, "owner", None) == owner: + return ep + if fallback is None: + fallback = ep + return fallback def _visible_image_endpoint_for_base(db, base: str, owner: str | None, model: str = ""): @@ -360,6 +719,92 @@ async def _fetch_result_image_b64(url: str) -> Optional[str]: return None +def _resolve_image_op_endpoint(db, owner, requested_base: str, chosen_model: str = "") -> Tuple[str, Optional[str]]: + """Resolve the provider base (with /v1) + api_key for a self-hosted image op. + + Mirrors inpaint_proxy: a client-supplied base must map to a registered, + owner-visible image endpoint (no direct use of request-body URLs). + """ + if requested_base: + ep = _visible_image_endpoint_for_base(db, requested_base, owner, chosen_model) + if not ep: + raise HTTPException(403, "Choose a registered image endpoint") + else: + ep = _first_visible_image_endpoint(db, owner) + if not ep: + raise HTTPException(400, "No image generation endpoint configured. Serve a diffusion model via Cookbook first.") + base = ep.base_url.rstrip("/") + if not base.endswith("/v1"): + base += "/v1" + return base, ep.api_key + + +async def _forward_seneca_image_op( + base: str, + api_key: Optional[str], + provider_path: str, + image_bytes: bytes, + fields: Dict[str, str], + *, + label: str, + timeout: float = 300.0, +) -> Dict[str, Any]: + """POST a single-image multipart op to the provider and return ``{image}``. + + The provider returns OpenAI-style ``{data:[{b64_json|url}]}`` or a raw + ``{image}``. ``request_id`` in ``fields`` keys the provider's progress route. + """ + import httpx + + url = base.rstrip("/") + provider_path + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + files = {"image": ("source.png", image_bytes, "image/png")} + _t0 = time.perf_counter() + try: + async with httpx.AsyncClient(timeout=timeout) as client: + r = await client.post(url, data=fields, files=files, headers=headers) + _provider_ms = (time.perf_counter() - _t0) * 1000.0 + except httpx.TimeoutException: + raise HTTPException(504, f"{label} timed out ({int(timeout)}s)") + except Exception as exc: + logger.warning("%s provider call failed: %s", label, exc) + raise HTTPException(503, f"{label} is temporarily unavailable (remote service unreachable).") + if r.status_code != 200: + logger.warning("%s provider status %s", label, r.status_code) + detail = f"{label} failed" + try: + j = r.json() + if isinstance(j, dict) and (j.get("error") or j.get("detail")): + detail = str(j.get("error") or j.get("detail"))[:200] + except Exception: + pass + raise HTTPException(r.status_code if r.status_code in (400, 404, 422) else 502, detail) + result = r.json() + raw_b64: Optional[str] = None + used_model = "" + if isinstance(result, dict): + used_model = str(result.get("model") or "") + data = result.get("data") + if isinstance(data, list) and data: + item = data[0] or {} + raw_b64 = item.get("b64_json") or None + if not raw_b64 and item.get("url"): + raw_b64 = await _fetch_result_image_b64(item["url"]) + if not raw_b64 and result.get("image"): + raw_b64 = result["image"] + if not raw_b64: + raise HTTPException(502, "Image endpoint returned no image") + logger.info( + "%s timing model_ms=%.1f total_ms=%.1f request_id=%s", + label, _provider_ms, + (time.perf_counter() - _t0) * 1000.0, fields.get("request_id") or "-", + ) + payload: Dict[str, Any] = {"image": raw_b64} + if used_model: + payload["model"] = used_model + return payload + + def setup_gallery_routes() -> APIRouter: router = APIRouter(tags=["gallery"]) @@ -602,7 +1047,7 @@ def setup_gallery_routes() -> APIRouter: # Use img2img endpoint if available, otherwise upscale via canvas on client try: - async with httpx.AsyncClient(timeout=120) as client: + async with httpx.AsyncClient(timeout=300) as client: resp = await client.post(f"{base_url}/images/upscale", json={ "image": b64, "scale": scale, }) @@ -645,7 +1090,7 @@ def setup_gallery_routes() -> APIRouter: base_url += "/v1" try: - async with httpx.AsyncClient(timeout=180) as client: + async with httpx.AsyncClient(timeout=300) as client: resp = await client.post(f"{base_url}/images/generations", json={ "prompt": prompt, "image": b64, @@ -1291,6 +1736,10 @@ def setup_gallery_routes() -> APIRouter: if not ok: raise HTTPException(400, f"Rejected endpoint URL: {reason}") chosen_model = (body.pop("_model", "") or "").strip() + # Optional client-supplied id so GET /api/image/edit-progress can poll the + # provider's step progress for this op. Ignored by providers that do not + # implement the progress contract. + progress_request_id = (body.get("request_id") or "").strip() api_key = None if not requested_base: db = SessionLocal() @@ -1338,8 +1787,10 @@ def setup_gallery_routes() -> APIRouter: raise HTTPException(500, "Pillow not installed on server") try: - img_bytes = base64.b64decode(body["image"]) - mask_bytes = base64.b64decode(body["mask"]) + img_bytes = _b64_bytes(body.get("image")) + if not body.get("mask"): + raise HTTPException(400, "A mask is required for OpenAI-compatible image edits") + mask_bytes = _b64_bytes(body.get("mask"), label="mask") source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA") mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L") # luminance # Build OpenAI mask: RGBA where alpha=255 means keep, 0 means regenerate. @@ -1390,7 +1841,11 @@ def setup_gallery_routes() -> APIRouter: } headers = {"Authorization": f"Bearer {api_key}"} try: - async with httpx.AsyncClient(timeout=120) as client: + logger.debug( + "inpaint_proxy provider edit -> prompt=%r model=%r size=%r base=%s", + body.get("prompt", ""), oa_model, size, base, + ) + async with httpx.AsyncClient(timeout=300) as client: if is_openrouter: # Reference-based editing, then composite locally so # pixels outside the user's mask remain unchanged. @@ -1410,7 +1865,16 @@ def setup_gallery_routes() -> APIRouter: r = await client.post(_join_checked_gallery_endpoint(base, "/images/edits"), headers=headers, data=data, files=files) if r.status_code != 200: logger.error("inpaint_proxy OpenAI edit: status %s", r.status_code) - raise HTTPException(r.status_code, "OpenAI edit failed") + _detail = "OpenAI edit failed" + try: + _j = r.json() + if isinstance(_j, dict): + _detail = _j.get("error") or _j.get("detail") or _detail + if isinstance(_detail, dict): + _detail = _detail.get("message") or _detail + except Exception: + pass + raise HTTPException(r.status_code, f"{_detail} (provider HTTP {r.status_code})") result = r.json() raw_b64 = None if result.get("data"): @@ -1441,13 +1905,16 @@ def setup_gallery_routes() -> APIRouter: blended = Image.composite(generated, source_png, mask_png) out_buf = io.BytesIO() blended.save(out_buf, format="PNG") - return {"image": base64.b64encode(out_buf.getvalue()).decode()} + return {"image": base64.b64encode(out_buf.getvalue()).decode(), "model": oa_model} except Exception as comp_err: # Never return a full-image edit when masking fails. logger.warning(f"Inpaint compose failed: {comp_err}") raise HTTPException(502, "Could not apply the edit within the selected region") except httpx.TimeoutException: - raise HTTPException(504, "OpenAI inpaint timed out (120s)") + raise HTTPException(504, "OpenAI inpaint timed out (300s)") + except httpx.TransportError as exc: + logger.warning("inpaint_proxy: OpenAI endpoint unreachable: %s", exc) + raise HTTPException(502, f"Image server unreachable: {type(exc).__name__}") # Self-hosted diffusion server path. Newer Odysseus image # wrappers expose the OpenAI-compatible /v1/images/edits @@ -1459,45 +1926,96 @@ def setup_gallery_routes() -> APIRouter: # supports multiple models per process. Harmless if ignored. if chosen_model: body["model"] = chosen_model - async with httpx.AsyncClient(timeout=240) as client: + async with httpx.AsyncClient(timeout=300) as client: + _t0 = time.perf_counter() + _restore_ms = [0.0] try: import base64, io from PIL import Image - img_bytes = base64.b64decode(body["image"]) - mask_bytes = base64.b64decode(body["mask"]) - # Normalize both inputs to PNG bytes. Local MLX and - # Diffusers wrappers expect white mask pixels to mean - # "edit this region", which matches the editor's mask. - source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA") - mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L") - src_buf = io.BytesIO() - source_png.save(src_buf, format="PNG") - mask_buf = io.BytesIO() - mask_png.save(mask_buf, format="PNG") + # Decode/resize/encode off the event loop: a 4000x3000 input + # costs ~0.2 s of CPU that would otherwise block every other + # request (and stack under concurrent edits). + _max_side = int(os.getenv("ODYSSEUS_IMAGE_EDIT_MAX_SIDE", "768") or 768) + _src_png, _mask_png, _model_w, _model_h, _orig_size = await asyncio.to_thread( + _prepare_local_edit, body.get("image"), body.get("mask"), _max_side + ) files = { - "image": ("source.png", src_buf.getvalue(), "image/png"), - "mask": ("mask.png", mask_buf.getvalue(), "image/png"), + "image": ("source.png", _src_png, "image/png"), + "mask": ("mask.png", _mask_png, "image/png"), } data = { "model": chosen_model or body.get("model") or "", "prompt": body.get("prompt", ""), - "size": f"{int(body.get('width') or source_png.width)}x{int(body.get('height') or source_png.height)}", + "size": f"{_model_w}x{_model_h}", "n": "1", } + if progress_request_id: + data["request_id"] = progress_request_id + # Let the caller's step budget through up to a cap (quality + # edits such as removal/background-replacement/expand pass 8 + # on purpose); only fall back to a moderate default when no + # count is requested. The size cap above is what keeps the + # op responsive. + _max_steps = int(os.getenv("ODYSSEUS_IMAGE_EDIT_MAX_STEPS", "8") or 8) + if body.get("steps"): + try: + data["steps"] = str(min(max(1, int(body["steps"])), _max_steps)) + except (TypeError, ValueError): + data["steps"] = "4" + else: + data["steps"] = "4" + + async def _restore_b64(_raw: str) -> str: + """Resize a provider PNG b64 back to the original size (off-loop).""" + if (_model_w, _model_h) == _orig_size: + return _raw + _rs = time.perf_counter() + try: + return await asyncio.to_thread( + _restore_local_b64, _raw, _orig_size, (_model_w, _model_h) + ) + finally: + _restore_ms[0] += (time.perf_counter() - _rs) * 1000.0 + logger.debug( + "inpaint_proxy self-hosted /images/edits -> prompt=%r model=%r " + "size=%r steps=%r endpoint=%s", + data.get("prompt"), data.get("model"), data.get("size"), + data.get("steps"), base, + ) + _prep_ms = (time.perf_counter() - _t0) * 1000.0 + + def _finish(_payload: Dict[str, Any]) -> Dict[str, Any]: + # model_ms is the provider round-trip (its own queue + + # diffusion time); prep_ms is our decode/resize; restore_ms + # is resizing the result back. total_ms is the whole op. + logger.info( + "inpaint-proxy timing prep_ms=%.1f model_ms=%.1f restore_ms=%.1f " + "total_ms=%.1f size=%s steps=%s model=%s request_id=%s", + _prep_ms, _provider_ms[0], _restore_ms[0], + (time.perf_counter() - _t0) * 1000.0, + data.get("size"), data.get("steps"), data.get("model"), + progress_request_id or "-", + ) + return _payload + + _t_post = time.perf_counter() r = await client.post(_join_checked_gallery_endpoint(base, "/images/edits"), data=data, files=files) + _provider_ms = [(time.perf_counter() - _t_post) * 1000.0] if r.status_code == 200: result = r.json() + _used_model = result.get("model") if isinstance(result, dict) else None + _used_model = _used_model or data.get("model") or "" if isinstance(result, dict) and result.get("data"): item = result["data"][0] if item.get("b64_json"): - return {"image": item["b64_json"]} + return _finish({"image": await _restore_b64(item["b64_json"]), "model": _used_model}) if item.get("url"): raw_b64 = await _fetch_result_image_b64(item["url"]) if raw_b64: - return {"image": raw_b64} + return _finish({"image": await _restore_b64(raw_b64), "model": _used_model}) if isinstance(result, dict) and result.get("image"): - return {"image": result["image"]} + return _finish({"image": await _restore_b64(result["image"]), "model": _used_model}) raise HTTPException(502, "Image edit endpoint returned no image") if r.status_code not in (404, 405): logger.warning("inpaint_proxy self-hosted edits: status %s", r.status_code) @@ -1528,16 +2046,158 @@ def setup_gallery_routes() -> APIRouter: r = await client.post(_join_checked_gallery_endpoint(base, "/images/inpaint"), json=body) if r.status_code != 200: logger.error("inpaint_proxy diffusion: status %s", r.status_code) - raise HTTPException(r.status_code, "Inpaint request failed") + detail = "Inpaint request failed" + try: + j = r.json() + if isinstance(j, dict): + detail = j.get("detail") or j.get("error") or detail + except Exception: + pass + raise HTTPException(r.status_code, f"{detail} (provider HTTP {r.status_code})") return r.json() except httpx.TimeoutException: raise HTTPException(504, "Inpaint request timed out (240s)") + except httpx.TransportError as exc: + logger.warning("inpaint_proxy: image endpoint unreachable: %s", exc) + raise HTTPException(502, f"Image server unreachable: {type(exc).__name__}") except HTTPException: raise except Exception: logger.exception("inpaint_proxy: request failed") raise HTTPException(502, "Inpaint request failed") + async def _resolve_and_decode(body: Dict[str, Any], user): + """Shared prep for the seneca single-image ops: validate endpoint + decode. + + Returns (base, api_key, image_bytes, chosen_model). The base64 decode + runs off the event loop (large images). + """ + requested_base = (body.pop("_endpoint", "") or "").rstrip("/") + if requested_base: + from src.url_safety import check_outbound_url + ok, reason = check_outbound_url( + requested_base, + block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise HTTPException(400, f"Rejected endpoint URL: {reason}") + chosen_model = (body.pop("_model", "") or "").strip() + db = SessionLocal() + try: + base, api_key = _resolve_image_op_endpoint(db, user, requested_base, chosen_model) + finally: + db.close() + image_bytes = await asyncio.to_thread(_b64_bytes, body.get("image")) + return base, api_key, image_bytes, chosen_model + + @router.post("/api/image/replace-background") + async def replace_background(request: Request): + """Proxy seneca's subject-mask background replacement (~/v1/images/replace-background).""" + user = require_privilege(request, "can_generate_images") + body = await request.json() + base, api_key, image_bytes, chosen_model = await _resolve_and_decode(body, user) + fields: Dict[str, str] = { + "prompt": str(body.get("prompt") or ""), + "steps": str(body.get("steps") or 4), + } + if body.get("request_id"): + fields["request_id"] = str(body["request_id"]).strip() + if body.get("harmonise") is not None: + fields["harmonise"] = "true" if body.get("harmonise") else "false" + if body.get("shadow") is not None: + fields["shadow"] = "true" if body.get("shadow") else "false" + if chosen_model: + fields["model"] = chosen_model + return await _forward_seneca_image_op( + base, api_key, "/images/replace-background", image_bytes, fields, + label="replace-background", + ) + + @router.post("/api/image/outpaint") + async def outpaint(request: Request): + """Proxy seneca's canvas outpaint/extend (~/v1/images/outpaint, alias /extend).""" + user = require_privilege(request, "can_generate_images") + body = await request.json() + base, api_key, image_bytes, chosen_model = await _resolve_and_decode(body, user) + fields: Dict[str, str] = { + "prompt": str(body.get("prompt") or ""), + "steps": str(body.get("steps") or 4), + } + for key in ("ratio", "size", "context"): + if body.get(key) is not None and str(body.get(key)) != "": + fields[key] = str(body[key]) + if body.get("request_id"): + fields["request_id"] = str(body["request_id"]).strip() + if chosen_model: + fields["model"] = chosen_model + return await _forward_seneca_image_op( + base, api_key, "/images/outpaint", image_bytes, fields, + label="outpaint", timeout=_outpaint_timeout(), + ) + + @router.post("/api/image/extend") + async def outpaint_extend(request: Request): + """Alias of /api/image/outpaint (the provider exposes /outpaint as /extend too).""" + return await outpaint(request) + + @router.get("/api/image/edit-progress") + async def edit_progress(request: Request): + """Poll a running image edit's provider-side step progress. + + Contract v1 (image-op progress): the provider exposes + ``GET /v1/images/progress/{request_id}`` -> ``{stage, step, steps, + eta_s, progress}``. This proxy resolves the configured image endpoint + (owner-scoped) and returns that JSON, or ``{"available": false}`` when + the provider has no progress route yet, so the client can keep an + indeterminate bar without breaking. + """ + import httpx + user = require_privilege(request, "can_generate_images") + request_id = (request.query_params.get("request_id") or "").strip() + if not request_id: + raise HTTPException(400, "Missing request_id") + # request_id is spliced into the outbound URL, so it must be a safe token + # (no path separators/host chars). The base itself is a DB-validated row. + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,80}", request_id): + raise HTTPException(400, "Invalid request_id") + requested_base = (request.query_params.get("base") or "").strip().rstrip("/") + endpoint_id = (request.query_params.get("endpoint_id") or "").strip() + db = SessionLocal() + try: + ep = None + if endpoint_id: + ep = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.id == endpoint_id), + ModelEndpoint, user, + ).first() + if ep is None and requested_base: + ep = _visible_image_endpoint_for_base(db, requested_base, user) + if ep is None: + ep = _first_visible_image_endpoint(db, user) + if ep is None: + return {"available": False, "reason": "no image endpoint configured"} + base = ep.base_url.rstrip("/") + finally: + db.close() + if not base.endswith("/v1"): + base += "/v1" + # Dynamic path segment (validated token above), so not routed through + # _join_checked_gallery_endpoint's exact-match whitelist. + url = f"{base}/images/progress/{request_id}" + try: + async with httpx.AsyncClient(timeout=10) as client: + r = await client.get(url) + if r.status_code == 200: + data = r.json() + if isinstance(data, dict): + data.setdefault("available", True) + return data + return {"available": True, "progress": data} + return {"available": False, "status": r.status_code} + except Exception as exc: + logger.debug("edit-progress poll failed: %s", exc) + return {"available": False, "reason": "provider unreachable"} + # ---- POST /api/image/harmonize — proper img2img call ---- # Earlier version routed through inpaint with a full-white mask, but # most backends interpret "100% mask coverage" as "regenerate from @@ -1689,10 +2349,10 @@ def setup_gallery_routes() -> APIRouter: headers["Authorization"] = f"Bearer {api_key}" last_err = None - # Cold-start SDXL inpaint can take 60-90s on first request (loading - # weights to GPU). 240s gives headroom for both that and a full - # 1024×1024 inference pass on slower setups. - async with httpx.AsyncClient(timeout=240) as client: + # Cold-start image servers can take 90-110s for a full pass (and + # 60-90s more on first request while weights load to GPU). 300s gives + # headroom for both on slower setups. + async with httpx.AsyncClient(timeout=300) as client: for path, kind, payload in candidates: _effective_base = base_root if path.startswith("/sdapi") else base target = _join_checked_gallery_endpoint(_effective_base, path) @@ -1755,7 +2415,7 @@ def setup_gallery_routes() -> APIRouter: from PIL import Image, ImageFilter import base64, io - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) img = Image.open(io.BytesIO(img_bytes)).convert("RGB") # Unsharp mask: radius=2, percent=amount*200, threshold=3 @@ -1776,6 +2436,9 @@ def setup_gallery_routes() -> APIRouter: image_b64 = body.get("image") if not image_b64: raise HTTPException(400, "No image provided") + _proxied = await _route_heavy_image_op("/api/image/denoise", body, label="AI denoise") + if _proxied is not None: + return _proxied try: strength = float(body.get("strength", 0.5)) except Exception: @@ -1788,7 +2451,7 @@ def setup_gallery_routes() -> APIRouter: except ImportError: raise HTTPException(500, "Server missing a required dependency") # Decode source image (RGB; Real-ESRGAN doesn't preserve alpha). - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) src = Image.open(io.BytesIO(img_bytes)).convert("RGB") try: patch_realesrgan_torchvision_compat() @@ -1827,6 +2490,9 @@ def setup_gallery_routes() -> APIRouter: image_b64 = body.get("image") if not image_b64: raise HTTPException(400, "No image provided") + _proxied = await _route_heavy_image_op("/api/image/upscale-local", body, label="AI upscale") + if _proxied is not None: + return _proxied try: scale = int(body.get("scale", 2)) except Exception: @@ -1838,7 +2504,7 @@ def setup_gallery_routes() -> APIRouter: import numpy as np except ImportError: raise HTTPException(500, "Server missing a required dependency") - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) src = Image.open(io.BytesIO(img_bytes)).convert("RGB") try: patch_realesrgan_torchvision_compat() @@ -1865,6 +2531,74 @@ def setup_gallery_routes() -> APIRouter: logger.warning("AI upscale failed", exc_info=True) return {"error": "AI upscale failed"} + # ---- POST /api/image/generate ---- + @router.post("/api/image/generate") + async def generate_image(request: Request): + """Text-to-image generation for the editor's Quick Edit 'Image' mode. + + Generates a brand-new image from a prompt using the admin-configured + image model, returned as base64 so the client can drop it in as a layer. + """ + import json as _json + user = require_privilege(request, "can_generate_images") + try: + body = await request.json() + except Exception: + body = {} + prompt = str((body or {}).get("prompt") or "").strip() + if not prompt: + raise HTTPException(400, "A prompt is required") + try: + width = int((body or {}).get("width") or 0) + height = int((body or {}).get("height") or 0) + except (TypeError, ValueError): + width = height = 0 + if height > width: + size = "1024x1536" + elif width > height: + size = "1536x1024" + else: + size = "1024x1024" + try: + from src.ai_interaction import do_generate_image + except Exception: + raise HTTPException(502, "Image generation is unavailable") + # The editor's model picker may choose a specific (cloud) image model; + # forward it so settings defaults only apply when none is selected. + model_choice = str((body or {}).get("model") or "").strip() + payload = {"prompt": prompt, "size": size} + if model_choice: + payload["model"] = model_choice + content = _json.dumps(payload) + result = await do_generate_image(content, owner=user, incognito=True) + if not isinstance(result, dict) or result.get("error"): + msg = (result.get("error") if isinstance(result, dict) else None) or "Image generation failed" + low = str(msg).lower() + # Plain-language status codes the editor maps to fixed messages: + # 503 -> generation is switched off on the server + # 502 -> the image server could not be reached (or failed) + if "disabled" in low or "switched off" in low or "not enabled" in low: + raise HTTPException(503, "Image generation is switched off on the server at the moment.") + if "timed out" in low or "timeout" in low: + raise HTTPException(504, f"Image server timed out: {msg}") + if "connection" in low or "connect" in low or "unreachable" in low: + raise HTTPException(502, f"Image server unreachable: {msg}") + # A configured image model that cannot be resolved means its endpoint + # did not answer, i.e. the image server is down, not a bad prompt. + if "no endpoint found with image model" in low: + raise HTTPException(502, f"Image server unreachable: {msg}") + raise HTTPException(502, msg) + image_url = result.get("image_url") or "" + b64 = image_url.split(",", 1)[1] if image_url.startswith("data:") else image_url + if not b64: + raise HTTPException(502, "Image generation returned no image") + return { + "image": b64, + "prompt": prompt, + "model": result.get("image_model"), + "size": result.get("image_size") or size, + } + # ---- POST /api/image/remove-bg ---- @router.post("/api/image/mask") async def smart_mask(request: Request): @@ -1876,6 +2610,9 @@ def setup_gallery_routes() -> APIRouter: """ require_privilege(request, "can_generate_images") body = await request.json() + _proxied = await _remote_smart_mask(body) + if _proxied is not None: + return _proxied image = _b64_to_pil_image(body.get("image") or "", mode="RGB") points = body.get("points") or [] box = body.get("box") @@ -1889,6 +2626,7 @@ def setup_gallery_routes() -> APIRouter: if not points and not box: raise HTTPException(400, "Provide at least one point, box, or object text") + _image_op_log_rss("mask", "start") backend = _load_sam_backend() torch = backend["torch"] processor = backend["processor"] @@ -1985,6 +2723,8 @@ def setup_gallery_routes() -> APIRouter: } if grounded: result["grounding"] = grounded + _image_op_log_rss("mask", "end") + _unload_image_models() return result except HTTPException: raise @@ -2010,11 +2750,14 @@ def setup_gallery_routes() -> APIRouter: body = await request.json() image_b64 = body.get("image") hint_b64 = body.get("hint_mask") + _proxied = await _remote_remove_background(body) + if _proxied is not None: + return _proxied from PIL import Image import base64, io - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) img = Image.open(io.BytesIO(img_bytes)).convert("RGBA") W, H = img.size @@ -2022,7 +2765,7 @@ def setup_gallery_routes() -> APIRouter: bbox = None if hint_b64: try: - hint_bytes = base64.b64decode(hint_b64) + hint_bytes = _b64_bytes(hint_b64, label="hint_mask") hint = Image.open(io.BytesIO(hint_bytes)).convert("L") # Resize the hint to match if dimensions disagree if hint.size != img.size: @@ -2046,6 +2789,7 @@ def setup_gallery_routes() -> APIRouter: else: crop = img + _image_op_log_rss("remove-bg", "start") try: from rembg import remove from starlette.concurrency import run_in_threadpool @@ -2061,6 +2805,9 @@ def setup_gallery_routes() -> APIRouter: cut = tmp except Exception: return {"error": "No background removal model available. Install rembg: pip install rembg"} + finally: + _image_op_log_rss("remove-bg", "end") + _unload_image_models() # Compose the cropped result back into a full-size transparent canvas. if bbox: @@ -2100,11 +2847,13 @@ def setup_gallery_routes() -> APIRouter: from PIL import Image, ImageFilter, ImageEnhance import numpy as np - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) img = Image.open(io.BytesIO(img_bytes)).convert("RGB") # Try GFPGAN first (AI face restoration) try: + if not _local_image_models_enabled(): + raise ImportError("local image models disabled") from gfpgan import GFPGANer import cv2 diff --git a/routes/model_routes.py b/routes/model_routes.py index 6aec3b9d9..9749e0bbd 100644 --- a/routes/model_routes.py +++ b/routes/model_routes.py @@ -473,7 +473,7 @@ def _truthy(value: str | None) -> bool: _ENDPOINT_KINDS = {"auto", "local", "api", "proxy"} _REFRESH_MODES = {"auto", "manual", "disabled"} -_MODEL_TOOL_MODES = {"none", "compact", "full"} +_MODEL_TOOL_MODES = {"none", "compact", "full", "spec"} _MODEL_TOOL_MODE_ALIASES = { "regular": "full", "odysseus_compact": "compact", @@ -553,10 +553,10 @@ def _endpoint_refresh_timeout(ep: Any, category: str) -> float: val = 0 if val > 0: return float(max(1, min(60, val))) - # llama.cpp and other local OpenAI-compatible servers can block briefly - # while warming/loading. A 2s local timeout makes working endpoints flicker - # offline before /v1/models is ready. - return 10.0 if category == "local" else 2.0 + # Keep the background refresh short so a single dead endpoint cannot stall + # the model picker. A timed-out refresh never clears a non-empty model + # cache, so a briefly-warming local server is simply retried next round. + return 2.0 def _manual_refresh_timeout(ep: Any, category: str, requested: Any = None) -> float: @@ -1511,6 +1511,9 @@ def setup_model_routes(model_discovery): _refresh_inflight = {"v": False, "done": None} # coarse single-flight guard _REFRESH_FAILURE_BASE = 300.0 _REFRESH_FAILURE_MAX = 3600.0 + # Even a user-triggered refresh must not re-probe an endpoint that just + # failed/timed out on every click; keep a short cooldown. + _REFRESH_FAILURE_MIN = 60.0 def _refresh_key(base: str, api_key: Optional[str], provider_auth_id: Optional[str] = None) -> str: # Session-backed endpoints carry no static key; include their auth id @@ -1558,14 +1561,20 @@ def setup_model_routes(model_discovery): if mode in ("manual", "disabled") and not force: return False, info fails = int(state.get("fail_count") or 0) - if fails and not force: + if fails: last_failure = float(state.get("last_failure") or 0.0) empty_local = ( not cached and category == "local" and str(getattr(ep, "id", "") or "").startswith("local-") ) - if now - last_failure < _failure_delay(fails, empty_local=empty_local): + delay = _failure_delay(fails, empty_local=empty_local) + if force: + # A user-triggered refresh gets a short cooldown instead of the + # long background backoff, but still skips a just-failed + # endpoint so repeated clicks cannot stall the picker. + delay = min(delay, _REFRESH_FAILURE_MIN) + if now - last_failure < delay: return False, info if cached and not force: interval = _endpoint_refresh_interval(ep, category) @@ -3089,4 +3098,10 @@ def setup_model_routes(model_discovery): router._should_refresh_endpoint = _should_refresh_endpoint router._search_endpoint_catalog = search_endpoint_catalog + # Exposed for tests: the in-memory backoff state, its key function, and the + # background refresh driver (probes endpoints in parallel with the short + # per-endpoint timeout). + router._refresh_state = _refresh_state + router._refresh_key = _refresh_key + router._refresh_caches_bg = _refresh_caches_bg return router diff --git a/scripts/generate_env_reference.py b/scripts/generate_env_reference.py index 06a8a4c64..ff0fe8974 100644 --- a/scripts/generate_env_reference.py +++ b/scripts/generate_env_reference.py @@ -914,6 +914,43 @@ VARIABLE_NOTES: dict[str, tuple[str, str, str]] = { "Overrides the source commit reported for runtime provenance, for builds " "that ship without a git directory.", ), + + # -- Photo editor and image operations ----------------------------------- + "ODYSSEUS_EDITOR_AGENT_BASES": ( + "Model routing and providers", USER, + "Comma-separated base URLs for the image-editor agent backends; a single " + "base URL in the model endpoint config wins over this list.", + ), + "ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS": ( + "Model routing and providers", USER, + "Set to 1, true, yes or on to allow local image-generation and editing " + "backends to be reachable as image providers.", + ), + "ODYSSEUS_IMAGE_OPS_BASE": ( + "Model routing and providers", USER, + "Base URL of the image-operations backend (inpaint, outpainting and style " + "transfer); a trailing slash is stripped before use.", + ), + "ODYSSEUS_IMAGE_EDIT_MAX_SIDE": ( + "Upload and media limits", USER, + "Maximum pixel side (width or height) an edited image is downscaled to " + "before it is sent to the image backend.", + ), + "ODYSSEUS_IMAGE_EDIT_MAX_STEPS": ( + "Upload and media limits", USER, + "Maximum number of iterative edit steps the image backend may perform in " + "a single request.", + ), + "ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT": ( + "Upload and media limits", USER, + "Seconds the outpainting proxy waits for the image backend before giving " + "up on a single request.", + ), + "ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT": ( + "Deployment and first run", INTERNAL, + "Seconds the server waits for in-flight requests and background loops to " + "finish during a graceful shutdown.", + ), } diff --git a/src/activity.py b/src/activity.py new file mode 100644 index 000000000..92c4098e5 --- /dev/null +++ b/src/activity.py @@ -0,0 +1,61 @@ +"""In-flight activity counters (heavy image ops) for safe restarts/observability. + +A deploy restart kills any in-flight long image op (a diffusion edit is 60-120 s), +so the operator needs to know whether the process is busy before issuing +``systemctl --user restart``. These counters are process-local and cheap. +""" +from __future__ import annotations + +import threading +import time +from contextlib import contextmanager +from typing import Dict + +_lock = threading.Lock() +_image_ops = 0 +_image_ops_by_name: Dict[str, int] = {} +_started_at = time.time() + + +def begin_image_op(name: str = "image") -> None: + global _image_ops + with _lock: + _image_ops += 1 + _image_ops_by_name[name] = _image_ops_by_name.get(name, 0) + 1 + + +def end_image_op(name: str = "image") -> None: + global _image_ops + with _lock: + _image_ops = max(0, _image_ops - 1) + if name in _image_ops_by_name: + _image_ops_by_name[name] -= 1 + if _image_ops_by_name[name] <= 0: + _image_ops_by_name.pop(name, None) + + +@contextmanager +def image_op(name: str = "image"): + begin_image_op(name) + try: + yield + finally: + end_image_op(name) + + +def activity() -> Dict[str, object]: + with _lock: + return { + "image_ops": _image_ops, + "image_ops_by_name": dict(_image_ops_by_name), + "uptime_s": round(time.time() - _started_at, 1), + } + + +def heavy_image_ops_active() -> int: + with _lock: + return _image_ops + + +def busy() -> bool: + return heavy_image_ops_active() > 0 diff --git a/src/agent_tools/__init__.py b/src/agent_tools/__init__.py index 7db8cffbb..7689f6ead 100644 --- a/src/agent_tools/__init__.py +++ b/src/agent_tools/__init__.py @@ -29,6 +29,7 @@ from .interaction_tools import AskUserTool, UpdatePlanTool from .model_interaction_tools import ChatWithModelTool, AskTeacherTool, ListModelsTool from .bg_job_tools import ManageBgJobsTool from .session_tools import CreateSessionTool, ListSessionsTool, SendToSessionTool, ManageSessionTool +from .editor_tools import PlanEditTool, RunEditorCommandTool, GetEditorScreenshotTool, AiEditImageTool from .admin_tools import ( ADMIN_TOOL_HANDLERS, do_manage_endpoints, do_manage_mcp, do_manage_webhooks, @@ -72,6 +73,10 @@ TOOL_HANDLERS = { "list_sessions": ListSessionsTool().execute, "send_to_session": SendToSessionTool().execute, "manage_session": ManageSessionTool().execute, + "plan_edit": PlanEditTool().execute, + "run_editor_command": RunEditorCommandTool().execute, + "get_editor_screenshot": GetEditorScreenshotTool().execute, + "ai_edit_image": AiEditImageTool().execute, } # Config/integration admin tools (manage_endpoints/mcp/webhooks/tokens/settings). TOOL_HANDLERS.update(ADMIN_TOOL_HANDLERS) diff --git a/src/agent_tools/editor_tools.py b/src/agent_tools/editor_tools.py new file mode 100644 index 000000000..d1fa4176e --- /dev/null +++ b/src/agent_tools/editor_tools.py @@ -0,0 +1,646 @@ +"""In-process agent tools for the scoped ``photo_editor`` capability family. + +These handlers run as the logged-in owner and drive the image editor over the +owner-scoped in-process bridge (``/api/editor/bridge/*``) plus the app's +existing image transforms (``/api/image/*``). They are only offered to the +model when the ``photo_editor`` family is active (the Quick Edit popup or an +explicit photo-edit chat turn), never in the general tool list. + +Loopback calls carry the internal token and impersonate ``owner`` (middleware +app.py), so the editor bridge resolves the same ``owner:session`` scope the +browser subscriber uses. +""" +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import logging +import re +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional + +import httpx + +from src.tools._common import _INTERNAL_BASE, _internal_headers + +logger = logging.getLogger(__name__) + + +def _provider_message() -> str: + try: + from src.ai_interaction import IMAGE_PROVIDER_REQUIRED_MESSAGE + + return IMAGE_PROVIDER_REQUIRED_MESSAGE + except Exception: + return ( + "Removing/adding objects needs an image generation provider; enable one " + "in Settings > Models." + ) + + +def _provider_available(owner: Optional[str]) -> bool: + try: + from src.ai_interaction import image_editing_provider_available + + return bool(image_editing_provider_available(owner)) + except Exception: + return True + +# Local diffusion/image servers can take 90-110s per image (8 steps) and +# longer on a cold start; the previous 120s default timed out mid-generation. +_EDITOR_TIMEOUT = 300.0 +# A mask covering less than this fraction of the image cannot drive a useful +# regional inpaint (tiny figures, empty SAM output): fall back to grounding or +# whole-image instruction mode instead. +_MIN_MASK_FRACTION = 0.002 +_AI_OPS = { + "remove-bg": "/api/image/remove-bg", + "upscale": "/api/image/upscale-local", + "denoise": "/api/image/denoise", + "enhance-face": "/api/image/enhance-face", + "inpaint": "/api/image/inpaint", + "generative": "/api/image/inpaint", +} +_PLACE_OP = "layers.placeImage" + + +def _parse(content: Any) -> Dict[str, Any]: + raw = content if isinstance(content, str) else json.dumps(content or {}) + try: + parsed = json.loads(raw) if raw.strip() else {} + except (ValueError, TypeError): + parsed = {} + return parsed if isinstance(parsed, dict) else {} + + +def _session(args: Dict[str, Any], ctx: Dict[str, Any]) -> str: + return str(args.get("session") or args.get("editor_session") or ctx.get("session_id") or "default") + + +async def _post(path: str, payload: Dict[str, Any], owner: Optional[str], timeout: float = _EDITOR_TIMEOUT) -> Dict[str, Any]: + headers = {**_internal_headers(owner), "Content-Type": "application/json"} + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post(f"{_INTERNAL_BASE}{path}", json=payload, headers=headers) + resp.raise_for_status() + try: + return resp.json() + except ValueError: + return {"raw": resp.text} + + +def _as_data_url(value: Any) -> Optional[str]: + if not isinstance(value, str) or not value: + return None + if value.startswith("data:"): + return value + if value.startswith("/"): + return None + return f"data:image/png;base64,{value}" + + +def _data_url_bytes(data_url: str) -> bytes: + _, _, b64 = data_url.partition(",") + return base64.b64decode(b64) + + +def _persist_png(image_bytes: bytes, owner: Optional[str], session_id: Optional[str], prompt: str, model: str) -> Dict[str, Any]: + """Save an edited/generated PNG and return {image_id, image_url, image_size}.""" + from core.database import GalleryImage, SessionLocal + from src.constants import GENERATED_IMAGES_DIR + + db = SessionLocal() + try: + root = Path(GENERATED_IMAGES_DIR).resolve() + root.mkdir(parents=True, exist_ok=True) + filename = f"{uuid.uuid4().hex[:12]}.png" + (root / filename).write_bytes(image_bytes) + new_id = str(uuid.uuid4()) + width = height = 0 + try: + from PIL import Image + + with Image.open(io.BytesIO(image_bytes)) as img: + width, height = img.size + except Exception: + pass + size = f"{width}x{height}" if width and height else "" + # The editor bridge uses an owner-scoped session id (e.g. "pewds:3") + # that is not a chat-sessions row. GalleryImage.session_id is a FK to + # sessions.id, so drop an unknown id instead of raising IntegrityError. + if session_id: + from core.database import Session as _ChatSession + + try: + known = db.query(_ChatSession.id).filter(_ChatSession.id == session_id).first() + except Exception: + known = None + if known is None: + session_id = None + derived = GalleryImage( + id=new_id, + filename=filename, + prompt=prompt or None, + caption=None, + model=model, + size=size or None, + quality=None, + tags=None, + ai_tags=None, + session_id=session_id, + album_id=None, + owner=owner, + file_hash=hashlib.sha256(image_bytes).hexdigest(), + file_size=len(image_bytes), + width=width or None, + height=height or None, + ) + db.add(derived) + db.commit() + return {"image_id": new_id, "image_url": f"/api/generated-image/{filename}", "image_size": size or None} + except Exception: + db.rollback() + raise + finally: + db.close() + + +async def _place_layer(data_url: str, name: str, owner: Optional[str], session: str, *, masked: bool = False, blend: str = "normal", opacity: float = 100.0, mode: Optional[str] = None) -> bool: + params: Dict[str, Any] = { + "dataUrl": data_url, + "name": name, + "masked": bool(masked), + "blend": blend or "normal", + "opacity": opacity, + } + # Forward the edit mode so the client can build a difference mask for + # instruction-mode results: the whole image was regenerated under an + # instruction prompt, so untouched pixels must stay pixel-identical (the + # client keeps only the changed region) instead of the new layer overwriting + # the entire canvas. Omit the keys entirely when no mode is given so the + # client keeps its current behavior (e.g. generate_image must not diff-mask). + if mode: + params["mode"] = mode + params["differenceMask"] = mode == "instruction" + if mode == "instruction": + # Exclude a thin edge band from the client difference mask: full-frame + # regenerations can shift pixels along the border. + params["edgeExcludePx"] = 3 + payload = { + "session": session, + "op": _PLACE_OP, + "params": params, + "wait": True, + "timeout": 60, + } + try: + data = await _post("/api/editor/bridge/command", payload, owner, timeout=90) + return data.get("ok") is not False + except Exception as exc: # pragma: no cover - defensive + logger.warning("editor layer placement failed: %s", exc) + return False + + +async def _fetch_selection_mask(owner: Optional[str], session: str) -> Optional[str]: + """Return the editor's current selection as a data-URL mask, or None. + + Generative ops (inpaint/generative) need a mask; the planner/judge may not + have supplied one, so fall back to whatever is selected in the editor. + """ + payload = { + "session": session, + "op": "selection.exportMask", + "params": {}, + "wait": True, + "timeout": 30, + } + try: + data = await _post("/api/editor/bridge/command", payload, owner, timeout=45) + except Exception as exc: # pragma: no cover - defensive + logger.warning("editor selection mask fetch failed: %s", exc) + return None + result = data.get("result") if isinstance(data, dict) else None + if isinstance(result, dict): + return _as_data_url(result.get("mask")) + return None + + +def _mask_white_fraction(data_url: Optional[str]) -> float: + """Fraction of a grayscale mask that is selected (white, >=128).""" + if not data_url: + return 0.0 + try: + from PIL import Image as _PILImage + + with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))).convert("L") as im: + total = im.width * im.height + if not total: + return 0.0 + hist = im.histogram() + return sum(hist[128:]) / total + except Exception: + return 0.0 + + +def _image_is_blank(data_url: str) -> bool: + """True when a result has an alpha channel that is essentially transparent.""" + try: + from PIL import Image as _PILImage + + with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))) as im: + if im.mode not in ("RGBA", "LA") and "transparency" not in im.info: + return False + alpha = im.convert("RGBA").getchannel("A") + hist = alpha.histogram() + total = sum(hist) + return total > 0 and (sum(hist[1:]) / total) < 0.001 + except Exception: + return False + + +def _rect_mask_data_url(image_data_url: str, box: List[float], *, pad_frac: float = 0.15) -> Optional[str]: + """Build a rectangular mask data URL from a [x1,y1,x2,y2] box, dilated. + + Used when the grounding detector finds an object but SAM returns an empty + mask (tiny figures): a dilated box guarantees a usable region. + """ + try: + from PIL import Image as _PILImage, ImageDraw + + with _PILImage.open(io.BytesIO(_data_url_bytes(image_data_url))) as im: + w, h = im.size + x1, y1, x2, y2 = (float(v) for v in box) + bw, bh = x2 - x1, y2 - y1 + x1, y1 = max(0.0, x1 - bw * pad_frac), max(0.0, y1 - bh * pad_frac) + x2, y2 = min(float(w), x2 + bw * pad_frac), min(float(h), y2 + bh * pad_frac) + mask = _PILImage.new("L", (w, h), 0) + ImageDraw.Draw(mask).rectangle([x1, y1, x2, y2], fill=255) + buf = io.BytesIO() + mask.save(buf, "PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() + except Exception: + return None + + +async def _ground_box(owner: Optional[str], image_data_url: str, label: str) -> Optional[List[float]]: + """Use the in-app grounding detector (OWL-ViT via /api/image/mask) for a box.""" + try: + data = await _post("/api/image/mask", {"image": image_data_url, "text": label}, owner, timeout=90) + except Exception as exc: # pragma: no cover - defensive + logger.warning("editor grounding box failed: %s", exc) + return None + box = data.get("bbox") if isinstance(data, dict) else None + if isinstance(box, (list, tuple)) and len(box) == 4: + try: + return [float(v) for v in box] + except Exception: + return None + return None + + +def _ensure_keep_clause(text: str) -> str: + """Append a keep-the-subject clause when the prompt lacks one.""" + body = str(text or "").strip() + if not body: + return body + if re.search(r"\b(keep|preserve|leave|unchanged|identical|same)\b", body, re.I): + return body + return body.rstrip(". ") + ". Keep the main subject and everything else in the image exactly the same." + + +def _mask_label(args: Dict[str, Any], prompt: str) -> str: + """Best-effort object noun for grounding a regional removal.""" + label = str(args.get("label") or args.get("object") or "").strip() + if label: + return label + m = re.search( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out)\s+(?:the\s+|a\s+|an\s+)?" + r"([a-z][a-z \-]{1,30}?)(?:\s+(?:on|in|at|from|to|near|behind|beside|by)\b|[.,;!?]|$)", + prompt or "", + re.I, + ) + if m: + noun = m.group(1).strip().lower() + if noun and noun not in {"background", "bg"}: + return noun + for noun in ("person", "people", "man", "woman", "car", "dog", "sign", "logo"): + if re.search(rf"\b{re.escape(noun)}\b", prompt or "", re.I): + return "person" if noun in {"person", "people", "man", "woman"} else noun + return "object" + + +class PlanEditTool: + """Plan an image edit as ordered editor operations (no side effects).""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + prompt = str(args.get("prompt") or "").strip() + if not prompt: + return "plan_edit: invalid", {"error": "plan_edit needs a non-empty `prompt`.", "exit_code": 1} + state = args.get("state") if isinstance(args.get("state"), dict) else {} + try: + data = await _post( + "/api/editor/plan", + {"prompt": prompt, "state": state, "mode": "plan", "session": _session(args, ctx)}, + ctx.get("owner"), + ) + except Exception as exc: + return "plan_edit: error", {"error": f"plan_edit failed: {exc}", "exit_code": 1} + steps = data.get("steps") or [] + reply = str(data.get("reply") or "") + return f"plan_edit: {len(steps)} step(s)", { + "output": reply or f"Planned {len(steps)} editor step(s).", + "reply": reply, + "planner": data.get("planner"), + "steps": steps, + "exit_code": 0, + } + + +class RunEditorCommandTool: + """Run a single validated editor op in the browser via the owner bridge.""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + op = str(args.get("op") or "").strip() + if not op: + return "run_editor_command: invalid", {"error": "run_editor_command needs `op`.", "exit_code": 1} + params = args.get("params") if isinstance(args.get("params"), dict) else {} + payload = { + "session": _session(args, ctx), + "op": op, + "params": params, + "wait": bool(args.get("wait", True)), + "timeout": float(args.get("timeout") or 60), + } + try: + data = await _post("/api/editor/bridge/command", payload, ctx.get("owner")) + except Exception as exc: + return f"run_editor_command: {op}", {"error": f"run_editor_command failed: {exc}", "exit_code": 1} + ok = data.get("ok") + result = data.get("result") + summary = json.dumps({"ok": ok, "error": data.get("error"), "result": result}, ensure_ascii=False) + return f"run_editor_command: {op}", { + "output": summary[:4000], + "result": result, + "screenshot": data.get("screenshot"), + "ok": ok, + "exit_code": 0 if ok is not False else 1, + } + + +class GetEditorScreenshotTool: + """Capture the current editor canvas as a data URL.""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + payload = { + "session": _session(args, ctx), + "op": "screenshot", + "params": {}, + "wait": True, + "timeout": float(args.get("timeout") or 30), + } + try: + data = await _post("/api/editor/bridge/command", payload, ctx.get("owner"), timeout=45) + except Exception as exc: + return "get_editor_screenshot: error", {"error": f"get_editor_screenshot failed: {exc}", "exit_code": 1} + result = data.get("result") if isinstance(data.get("result"), dict) else {} + shot = data.get("screenshot") or result.get("dataUrl") + if not shot: + return "get_editor_screenshot: unavailable", { + "error": "No editor screenshot was returned (is the editor open?).", + "exit_code": 1, + } + return "get_editor_screenshot", {"output": "Captured the editor canvas.", "screenshot": shot, "exit_code": 0} + + +class AiEditImageTool: + """Apply an AI image transform (remove-bg/upscale/denoise/enhance/inpaint).""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + _t = {"start": time.perf_counter()} + op = str(args.get("op") or "").strip().lower() + if op not in _AI_OPS: + return "ai_edit_image: invalid", { + "error": f"ai_edit_image `op` must be one of {sorted(_AI_OPS)}.", + "exit_code": 1, + } + owner = ctx.get("owner") + if op in ("inpaint", "generative") and not _provider_available(owner): + return "ai_edit_image: provider required", { + "error": _provider_message(), + "failure_kind": "image_provider_unavailable", + "needs_provider": True, + "exit_code": 1, + } + session = _session(args, ctx) + image = _as_data_url(args.get("image")) + if image is None and isinstance(args.get("image"), str) and args["image"].startswith("/"): + # A same-origin gallery/attachment URL: fetch it (owner-scoped) as bytes. + try: + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.get(f"{_INTERNAL_BASE}{args['image']}", headers=_internal_headers(owner)) + resp.raise_for_status() + image = _as_data_url("data:image/png;base64," + base64.b64encode(resp.content).decode()) + except Exception: + image = None + used_canvas = False + if not image: + shot = await GetEditorScreenshotTool().execute(json.dumps({"session": session}), ctx) + image = (shot[1] or {}).get("screenshot") if isinstance(shot, tuple) else None + used_canvas = bool(image) + if not image: + return "ai_edit_image: no image", { + "error": "ai_edit_image needs an `image` (data URL or same-origin URL) or an open editor.", + "exit_code": 1, + } + _t["image"] = time.perf_counter() + payload: Dict[str, Any] = {"image": image} + model_choice = str(args.get("model") or "").strip() + mask_used = False + place_mode: Optional[str] = None + if op == "upscale": + payload["scale"] = int(args.get("scale") or 2) + elif op == "denoise": + payload["strength"] = float(args.get("strength") or 0.5) + elif op in ("inpaint", "generative"): + payload["prompt"] = str(args.get("prompt") or "").strip() + if model_choice: + # inpaint_proxy reads `_model` to route to a specific model. + payload["_model"] = model_choice + mode = str(args.get("mode") or "instruction").strip().lower() + _steps = int(args.get("steps") or 0) + if _steps > 0: + payload["steps"] = _steps + mask_used = False + mask = None + if mode == "masked": + mask = _as_data_url(args.get("mask")) + if not mask: + mask = await _fetch_selection_mask(owner, session) + # An empty/tiny SAM mask on small figures is useless: drop it. + if mask and _mask_white_fraction(mask) < _MIN_MASK_FRACTION: + mask = None + if not mask: + # Fall back to the grounding detector (OWL-ViT): a dilated + # box mask around the matching object. + box = await _ground_box(owner, image, _mask_label(args, payload["prompt"])) + if box: + candidate = _rect_mask_data_url(image, box, pad_frac=0.15) + if candidate and _mask_white_fraction(candidate) >= _MIN_MASK_FRACTION: + mask = candidate + if mask: + payload["mask"] = mask + payload["mode"] = "masked" + mask_used = True + else: + # Nothing usable to mask -> whole-image instruction fallback + # rather than placing a transparent/empty layer. + mode = "instruction" + if mode != "masked": + # Instruction mode: whole-image edit with no tight mask. Send a + # full-white mask so the edit endpoint accepts the request and + # regenerates the whole frame under the instruction prompt, which + # names what must stay unchanged. This avoids the mask-compositing + # halos that loose selections caused. + payload["mode"] = "instruction" + payload.pop("mask", None) + payload["prompt"] = _ensure_keep_clause(payload["prompt"]) + try: + from PIL import Image as _PILImage + + _raw = _data_url_bytes(image) + _w, _h = _PILImage.open(io.BytesIO(_raw)).size + _mbuf = io.BytesIO() + _PILImage.new("L", (_w, _h), 255).save(_mbuf, "PNG") + payload["mask"] = "data:image/png;base64," + base64.b64encode(_mbuf.getvalue()).decode() + except Exception: + pass + place_mode = "masked" if mask_used else "instruction" + _t["prep"] = time.perf_counter() + try: + data = await _post(_AI_OPS[op], payload, owner) + except Exception as exc: + logger.info( + "image-op ai_edit_image op=%s stage=provider FAILED image_ms=%.1f prep_ms=%.1f total_ms=%.1f err=%s", + op, (_t["image"] - _t["start"]) * 1000.0, (_t["prep"] - _t["image"]) * 1000.0, + (time.perf_counter() - _t["start"]) * 1000.0, exc, + ) + return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) failed: {exc}", "exit_code": 1} + _t["provider"] = time.perf_counter() + result = data.get("image") or data.get("data") + data_url = _as_data_url(result) + if not data_url: + return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) returned no image.", "exit_code": 1} + if _image_is_blank(data_url): + return f"ai_edit_image: {op}", { + "error": f"ai_edit_image ({op}) produced an empty layer.", + "failure_kind": "empty_result", + "exit_code": 1, + } + + info: Dict[str, Any] = { + "output": f"Applied AI edit '{op}'" + (" to the current canvas." if used_canvas else "."), + "op": op, + "model": data.get("model") or model_choice or "", + "exit_code": 0, + } + try: + info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), f"ai_edit_image:{op}", f"ai_edit_image:{op}")) + except Exception as exc: + logger.warning("ai_edit_image persist failed: %s", exc) + _t["persist"] = time.perf_counter() + if bool(args.get("place", True)): + placed = await _place_layer( + data_url, + f"AI {op}", + owner, + session, + masked=mask_used, + blend=str(args.get("blend") or "normal"), + opacity=float(args.get("opacity") or 100), + mode=place_mode, + ) + info["placed"] = placed + if placed: + info["output"] += " Placed the result as a new layer." + _t["place"] = time.perf_counter() + logger.info( + "image-op ai_edit_image op=%s mode=%s canvas=%s image_ms=%.1f mask_ms=%.1f " + "model_ms=%.1f persist_ms=%.1f place_ms=%.1f total_ms=%.1f", + op, place_mode or "instruction", used_canvas, + (_t["image"] - _t["start"]) * 1000.0, + (_t["prep"] - _t["image"]) * 1000.0, + (_t["provider"] - _t["prep"]) * 1000.0, + (_t["persist"] - _t["provider"]) * 1000.0, + (_t["place"] - _t["persist"]) * 1000.0, + (_t["place"] - _t["start"]) * 1000.0, + ) + return f"ai_edit_image: {op}", info + + +class GenerateImageLayerTool: + """Generate an image with the configured image model and place it as a layer.""" + + async def execute(self, content: str, ctx: dict): + args = _parse(content) + prompt = str(args.get("prompt") or "").strip() + if not prompt: + return "generate_image: invalid", {"error": "generate_image needs a non-empty `prompt`.", "exit_code": 1} + owner = ctx.get("owner") + if not _provider_available(owner): + return "generate_image: provider required", { + "error": _provider_message(), + "failure_kind": "image_provider_unavailable", + "needs_provider": True, + "exit_code": 1, + } + session = _session(args, ctx) + payload: Dict[str, Any] = {"prompt": prompt} + for key in ("model", "size", "quality"): + if args.get(key): + payload[key] = args[key] + try: + from src.ai_interaction import do_generate_image + + data = await do_generate_image( + json.dumps(payload, ensure_ascii=False), + session_id=session, + owner=owner, + incognito=True, + ) + except Exception as exc: + return "generate_image: error", {"error": f"generate_image failed: {exc}", "exit_code": 1} + result = data.get("image_url") or data.get("image") or data.get("data") + data_url = _as_data_url(result) + if not data_url: + return "generate_image", {"error": "generate_image returned no image.", "exit_code": 1} + info: Dict[str, Any] = { + "output": f"Generated an image for: {prompt[:120]}", + "image_size": data.get("image_size"), + "model": data.get("image_model") or str(args.get("model") or ""), + "exit_code": 0, + } + try: + info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), prompt, "generate_image")) + except Exception as exc: + logger.warning("generate_image persist failed: %s", exc) + if bool(args.get("place", True)): + placed = await _place_layer( + data_url, + str(args.get("name") or "Generated image"), + owner, + session, + masked=bool(args.get("mask")), + blend=str(args.get("blend") or "normal"), + opacity=float(args.get("opacity") or 100), + ) + info["placed"] = placed + if placed: + info["output"] += " Placed it as a new layer." + return "generate_image", info diff --git a/src/ai_interaction.py b/src/ai_interaction.py index c8f7979ca..dd772eff1 100644 --- a/src/ai_interaction.py +++ b/src/ai_interaction.py @@ -965,7 +965,7 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O elif action == "open_panel": # Open a top-level panel/modal: documents/library, gallery, - # calendar, email, sessions, notes, memories, skills, settings, theme, cookbook. + # calendar, email, sessions, notes, tasks, memories, skills, settings, theme, cookbook. panel = parts[1].lower() if len(parts) > 1 else "" view = "" view_label = "" @@ -993,6 +993,8 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O "note": "notes", "todo": "notes", "todos": "notes", + "tasks": "tasks", + "task": "tasks", "memories": "memories", "memory": "memories", "brain": "memories", @@ -1010,7 +1012,7 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O } target = _panel_aliases.get(panel) if not target: - return {"error": f"Unknown panel '{panel}'. Valid: documents, gallery, calendar, email, sessions, notes, memories, skills, settings, theme, cookbook."} + return {"error": f"Unknown panel '{panel}'. Valid: documents, gallery, calendar, email, sessions, notes, tasks, memories, skills, settings, theme, cookbook."} if target == "cookbook": cookbook_views = { "models": ("Search", "models"), "model": ("Search", "models"), @@ -1155,7 +1157,50 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O # Image generation # --------------------------------------------------------------------------- -async def do_generate_image(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict: +# Message used whenever a generative image edit (object add/remove/replace, +# image generation) is requested but no image provider is configured. The +# photo-editor agent and the image tools surface this verbatim so the user gets +# a plain, actionable explanation instead of a silent substitute edit. +IMAGE_PROVIDER_REQUIRED_MESSAGE = ( + "Removing/adding objects needs an image generation provider; enable one in " + "Settings > Models." +) + + +def image_editing_provider_available(owner: Optional[str] = None) -> bool: + """True when an enabled, owner-visible image endpoint can serve AI edits. + + Generative operations (object add/remove/replace, image generation) require + a configured image model. Local transforms (remove-bg, upscale, denoise, + enhance-face) do not, but the photo-editor agent gates the whole generative + request on this so it never falls back to deterministic approximations. + """ + try: + from src.settings import load_settings + + if not load_settings().get("image_gen_enabled", True): + return False + except Exception: + pass + try: + from src.database import SessionLocal, ModelEndpoint + from src.auth_helpers import owner_filter + + db = SessionLocal() + try: + q = db.query(ModelEndpoint).filter( + ModelEndpoint.is_enabled == True, # noqa: E712 + ModelEndpoint.model_type == "image", + ) + q = owner_filter(q, ModelEndpoint, owner) + return q.first() is not None + finally: + db.close() + except Exception: + return False + + +async def do_generate_image(content: str, session_id: Optional[str] = None, owner: Optional[str] = None, *, incognito: bool = False) -> Dict: """Generate an image using an image-capable model (e.g. gpt-image-1). Content format: @@ -1255,6 +1300,14 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne async def _resolve_image_model(model_name: str): def _call(): + # Try a plain by-name lookup first. Gateway endpoints (OpenRouter) + # register as `llm` yet serve image models, and the image-typed + # query's last-resort fallback would otherwise hijack a cloud model + # onto the local image endpoint (gpt-5-image -> qwen-image-edit). + try: + return _resolve_model(model_name, owner=owner) + except ValueError: + pass try: return _resolve_model(model_name, owner=owner, model_type="image") except TypeError as exc: @@ -1314,7 +1367,7 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne else: payload["quality"] = "medium" - logger.info(f"Image generation: model={model_id}, size={size}, quality={quality}, prompt={prompt[:80]}") + logger.info(f"Image generation: model={model_id}, size={size}, quality={quality}, prompt={'[private]' if incognito else prompt[:80]}") try: # GPT image models can take 30-120s+ depending on quality @@ -1365,18 +1418,22 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne logger.warning(f"Failed to save gallery record: {_ge}") return "" - # GPT image models always return b64_json; DALL-E may return url - if img.get("b64_json"): + def _store_image(image_bytes): + if incognito: + return 'data:image/png;base64,' + base64.b64encode(image_bytes).decode('ascii'), None img_dir = Path(GENERATED_IMAGES_DIR) img_dir.mkdir(parents=True, exist_ok=True) filename = f"{uuid.uuid4().hex[:12]}.png" img_path = img_dir / filename - img_path.write_bytes(base64.b64decode(img.get("b64_json"))) + img_path.write_bytes(image_bytes) if owner: from src.generated_images import register_generated_image register_generated_image(filename, owner) - image_url = f"/api/generated-image/{filename}" - image_id = _save_to_gallery(filename) + return f"/api/generated-image/{filename}", _save_to_gallery(filename) + + # Private results stay in the response, never in the gallery or on disk. + if img.get("b64_json"): + image_url, image_id = _store_image(base64.b64decode(img['b64_json'])) elif img.get("url"): # Download external URL and save locally (DALL-E returns temp URLs) @@ -1390,16 +1447,7 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne try: dl_resp = httpx.get(result_url, timeout=60) if dl_resp.status_code == 200: - img_dir = Path(GENERATED_IMAGES_DIR) - img_dir.mkdir(parents=True, exist_ok=True) - filename = f"{uuid.uuid4().hex[:12]}.png" - img_path = img_dir / filename - img_path.write_bytes(dl_resp.content) - if owner: - from src.generated_images import register_generated_image - register_generated_image(filename, owner) - image_url = f"/api/generated-image/{filename}" - image_id = _save_to_gallery(filename) + image_url, image_id = _store_image(dl_resp.content) else: image_url = result_url # fallback to external URL except Exception as _dl_e: @@ -1436,6 +1484,7 @@ async def do_edit_image( size: str = "auto", quality: str = "medium", progress_callback: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None, + incognito: bool = False, ) -> Dict: """Edit an uploaded image using the configured image endpoint.""" import base64 @@ -1509,7 +1558,7 @@ async def do_edit_image( request_id = uuid.uuid4().hex payload["request_id"] = request_id - logger.info("Image edit: model=%s, size=%s, quality=%s, image=%s, prompt=%s", model_id, size, quality, path.name, prompt[:80]) + logger.info("Image edit: model=%s, size=%s, quality=%s, image=%s, prompt=%s", model_id, size, quality, '[private]' if incognito else path.name, '[private]' if incognito else prompt[:80]) def _save_edited_image_to_gallery(filename: str) -> str: try: @@ -1539,6 +1588,9 @@ async def do_edit_image( from PIL import Image with Image.open(BytesIO(image_bytes)) as output: size = f"{output.width}x{output.height}" + image_mime = Image.MIME.get(output.format, 'image/png') + if incognito: + return 'data:' + image_mime + ';base64,' + base64.b64encode(image_bytes).decode('ascii'), '' img_dir = Path(GENERATED_IMAGES_DIR) img_dir.mkdir(parents=True, exist_ok=True) filename = f"{uuid.uuid4().hex[:12]}{suffix}" diff --git a/src/editor_agent_model.py b/src/editor_agent_model.py new file mode 100644 index 000000000..88b638b29 --- /dev/null +++ b/src/editor_agent_model.py @@ -0,0 +1,498 @@ +"""Editor agent/planner model resolution. + +The image-editor Quick Edit planner (`/api/editor/plan`, `/api/editor/agent`) +runs on an explicitly configured local vision model when one is reachable, +with the configured chat model as a fallback. This module owns: + + * the local model base URLs / model id (settings-overridable), + * an ordered candidate list (per-request ``agent_model`` override, then the + local Qwen3.5-9B, then the hosted default/utility chat model), + * a vision fallback chain: local model WITH the canvas screenshot, else a + text-only call whose image is described by a separate VL caption model, + * a lightweight reachability probe for the configured local bases. + +Everything is best-effort: when no local server is reachable the planner +transparently falls back to the previously-used hosted chat model, so the +editor agent never regresses to "no plan". +""" + +from __future__ import annotations + +import contextvars +import json +import logging +import os +import re +import threading +from typing import Any, Dict, List, Optional, Tuple + +import httpx + +logger = logging.getLogger(__name__) + +# Local endpoints are opt-in through settings or ODYSSEUS_EDITOR_AGENT_BASES. +DEFAULT_LOCAL_BASES = [] +DEFAULT_LOCAL_MODEL = "Qwen/Qwen3.5-9B" + +# Vision caption model used when the editor-agent model cannot accept images. +DEFAULT_CAPTION_BASE = "" +DEFAULT_CAPTION_MODEL = "Qwen/Qwen3.5-9B" + +# Served-model-id cache per base (populated on first use). The local servers 400 +# with "Failed to load model" when asked for an id they do not serve (e.g. the +# client's default "Qwen3.5-9B-MLX-8bit", or a stale caption model), so we +# resolve the requested id against GET {base}/models and fall back to one that +# is actually served. +_MODELS_CACHE: Dict[str, set] = {} + +_CAPTION_INSTRUCTION = ( + "Describe this image in one or two sentences for a photo editor: the main " + "subject, the background/scene, lighting, dominant colours, and any obvious " + "flaws (blown highlights, colour cast, clutter, blur)." +) + + +def _setting(key: str, default: Any = "") -> Any: + try: + from src.settings import load_settings + + value = load_settings().get(key) + return value if value not in (None, "") else default + except Exception: + return default + + +def local_bases() -> List[str]: + override = _setting("editor_agent_base_url") + if override: + return [str(override)] + env = os.getenv("ODYSSEUS_EDITOR_AGENT_BASES") + if env: + bases = [b.strip() for b in env.split(",") if b.strip()] + if bases: + return bases + return list(DEFAULT_LOCAL_BASES) + + +def local_model() -> str: + return str(_setting("editor_agent_model", DEFAULT_LOCAL_MODEL)) + + +def caption_base() -> str: + return str(_setting("editor_caption_base_url", DEFAULT_CAPTION_BASE)) + + +def caption_model() -> str: + return str(_setting("editor_caption_model", DEFAULT_CAPTION_MODEL)) + + +# Per-request / per-plan context (contextvars keep this correct under the +# async event loop without sharing state between concurrent editor turns). +_override: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( + "editor_agent_model_override", default=None +) +_last_model: contextvars.ContextVar[Optional[Tuple[str, str]]] = contextvars.ContextVar( + "editor_agent_last_model", default=None +) + + +def set_override(value: Optional[str]) -> None: + _override.set(value or None) + + +def get_override() -> Optional[str]: + return _override.get() + + +def set_last_model(model: str, source: str) -> None: + _last_model.set((model, source)) + + +def get_last_model() -> Optional[Tuple[str, str]]: + return _last_model.get() + + +def _chat_url(base: str) -> str: + b = str(base or "").rstrip("/") + if b.endswith("/chat/completions"): + return b + return b + "/chat/completions" + + +def _split_override(override: str) -> Tuple[str, str]: + """Parse ``base::model`` (or a bare model id).""" + raw = str(override or "").strip() + if "::" in raw: + base, model = raw.split("::", 1) + return base.strip(), model.strip() + return "", raw + + +def _has_image(messages: List[dict]) -> bool: + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "image_url": + return True + return False + + +def _first_image(messages: List[dict]) -> Optional[str]: + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "image_url": + url = (part.get("image_url") or {}).get("url") + if isinstance(url, str): + return url + return None + + +def _strip_images(messages: List[dict]) -> List[dict]: + out: List[dict] = [] + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + text = "\n".join( + str(part.get("text") or "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ) + out.append({"role": msg.get("role", "user"), "content": text}) + else: + out.append({"role": msg.get("role", "user"), "content": content}) + return out + + +def _with_caption(messages: List[dict], caption: str) -> List[dict]: + note = f"\n\n[Canvas screenshot caption: {caption}]" + out = [dict(m) for m in messages] + for msg in reversed(out): + if msg.get("role") == "user": + msg["content"] = str(msg.get("content") or "") + note + break + return out + + +async def _post_chat( + base: str, + model: str, + messages: List[dict], + headers: Optional[dict], + timeout: float, + max_tokens: int = 1024, + response_format: Optional[dict] = None, +) -> Optional[str]: + if not base or not model: + return None + payload: Dict[str, Any] = { + "model": model, + "messages": messages, + "temperature": 0.2, + "max_tokens": max_tokens, + "stream": False, + } + # Constrain the planner to a strict JSON ops schema (structured outputs / + # grammar) so the model can only emit valid editor ops. Unknown servers + # ignore the field; the server-side schema validation is the real enforcer. + if isinstance(response_format, dict): + payload["response_format"] = response_format + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post(_chat_url(base), json=payload, headers=headers or {}) + if resp.status_code != 200: + logger.debug( + "editor agent chat %s -> %s (%s)", base, resp.status_code, resp.text[:200] + ) + return None + data = resp.json() + except Exception as exc: + logger.debug("editor agent chat %s failed: %s", base, exc) + return None + try: + return data["choices"][0]["message"]["content"] + except Exception: + return None + + +async def _served_models(base: str, timeout: float = 3.0) -> set: + """Return the set of model ids a base serves (cached; empty on failure).""" + if not base: + return set() + cached = _MODELS_CACHE.get(base) + if cached is not None: + return cached + ids: set = set() + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.get(base.rstrip("/") + "/models") + if resp.status_code == 200: + ids = { + str(m.get("id")) + for m in (resp.json().get("data") or []) + if isinstance(m, dict) and m.get("id") + } + except Exception: + ids = set() + _MODELS_CACHE[base] = ids + return ids + + +async def _resolve_model(base: str, requested: str, fallback: str) -> str: + """Return a model id ``base`` actually serves. + + If the requested id is served, keep it; otherwise use ``fallback`` (when + served) or any served id. When the served list is unknown (server down), + keep the requested id (or ``fallback``). + """ + requested = str(requested or "").strip() + fallback = str(fallback or "").strip() + ids = await _served_models(base) + if ids: + if requested and requested in ids: + return requested + if fallback in ids: + return fallback + return sorted(ids)[0] + return requested or fallback + + +async def caption_image(data_url: str, owner: Optional[str] = None, timeout: float = 90.0) -> Optional[str]: + if not isinstance(data_url, str) or not data_url.startswith("data:image/"): + return None + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": _CAPTION_INSTRUCTION}, + {"type": "image_url", "image_url": {"url": data_url}}, + ], + } + ] + base = caption_base() + if not base: + return None + model = await _resolve_model(base, caption_model(), local_model()) + return await _post_chat(base, model, messages, None, timeout, max_tokens=200) + + +_DONE_RE = re.compile(r"^\s*DONE\b", re.I) +_RETRY_RE = re.compile(r"RETRY:?\s*(.+)", re.I | re.S) + + +def _parse_self_check(text: str) -> Dict[str, Any]: + s = str(text or "").strip() + if _DONE_RE.match(s): + return {"done": True, "issues": [], "retry_hint": "", "text": s} + m = _RETRY_RE.search(s) + if m or re.search(r"\bretry\b", s, re.I): + hint = (m.group(1).strip() if m else s) or s + return {"done": False, "issues": [hint], "retry_hint": hint, "text": s} + return {"done": True, "issues": [], "retry_hint": "", "text": s} + + +async def self_check( + before: Any, + after: Any, + prompt: str, + model: Optional[str] = None, + owner: Optional[str] = None, + timeout: float = 30.0, +) -> Dict[str, Any]: + """Same-origin VL self-check: compare BEFORE vs AFTER against the intent. + + The browser cannot call the local VL server directly (it sends no CORS + headers), so ``/api/editor/self-check`` proxies here. Returns + ``{done, issues, retry_hint, text}``; on any failure (bad input, the model + is unreachable, or a timeout) it degrades to ``done=True`` with + ``reason="unavailable"`` so the client never gets stuck retrying. + """ + if not ( + isinstance(before, str) and before.startswith("data:image/") + and isinstance(after, str) and after.startswith("data:image/") + ): + return {"done": True, "issues": [], "retry_hint": "", + "reason": "unavailable", "text": "DONE"} + bases = local_bases() or list(DEFAULT_LOCAL_BASES) + if not bases: + return {"done": True, "issues": [], "retry_hint": "", + "reason": "unavailable", "text": "DONE"} + base = bases[0] + mdl = await _resolve_model(base, str(model or local_model()), local_model()) + messages = [{ + "role": "user", + "content": [ + {"type": "text", "text": str(prompt or "")}, + {"type": "image_url", "image_url": {"url": before}}, + {"type": "image_url", "image_url": {"url": after}}, + ], + }] + text = await _post_chat(base, mdl, messages, None, timeout, max_tokens=140) + if not text: + return {"done": True, "issues": [], "retry_hint": "", + "reason": "unavailable", "text": "DONE"} + out = _parse_self_check(text) + out["model"] = mdl + out["source"] = "local" + return out + + +def _candidates(owner: Optional[str], override: Optional[str]) -> List[Dict[str, Any]]: + cands: List[Dict[str, Any]] = [] + if override: + base, model = _split_override(override) + if model: + cands.append( + {"base": base or local_bases()[0], "model": model, "headers": None, "source": "override"} + ) + for base in local_bases(): + cands.append({"base": base, "model": local_model(), "headers": None, "source": "local"}) + try: + from src.endpoint_resolver import resolve_endpoint + + for prefix in ("utility", "default"): + try: + url, model, headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model: + cands.append({"base": url, "model": model, "headers": headers, "source": "hosted"}) + break + except Exception: + pass + return cands + + +async def plan_with_editor_agent( + messages: List[dict], + owner: Optional[str] = None, + override: Optional[str] = None, + timeout: float = 90.0, + response_format: Optional[dict] = None, +) -> Optional[Tuple[str, str, str]]: + """Return (raw_plan_text, model_used, source) or None. + + Tries the local model WITH the screenshot, then the same model text-only + with a VL caption of the screenshot, then the hosted chat model. + + ``response_format`` (a JSON-schema structured-output constraint) is sent + ONLY to the local editor-agent bases, so a hosted fallback model that does + not support the field never rejects the request. + """ + has_img = _has_image(messages) + text_messages = _strip_images(messages) if has_img else messages + image = _first_image(messages) if has_img else None + caption: Optional[str] = None + _local = {str(b).rstrip("/") for b in local_bases()} + for cand in _candidates(owner, override): + base, model = cand["base"], cand["model"] + headers = cand.get("headers") + is_local = cand["source"] == "local" or str(base).rstrip("/") in _local + rf = response_format if (response_format and is_local) else None + extra = {"response_format": rf} if rf else {} + if has_img: + raw = await _post_chat(base, model, messages, headers, timeout, **extra) + if raw: + return raw, model, cand["source"] + if caption is None and image: + caption = await caption_image(image, owner) + if caption: + raw = await _post_chat( + base, model, _with_caption(text_messages, caption), headers, timeout, **extra, + ) + if raw: + return raw, model, cand["source"] + "+caption" + else: + raw = await _post_chat( + base, model, text_messages, headers, timeout, **extra + ) + if raw: + return raw, model, cand["source"] + return None + + +async def list_editor_agent_models(owner: Optional[str] = None) -> List[Dict[str, Any]]: + """Models available to the editor agent, for the model picker. + + Lists the local Qwen3.5-9B (one entry per reachable base) plus the hosted + default/utility chat model that the planner falls back to. + """ + models: List[Dict[str, Any]] = [] + seen: set = set() + for base in local_bases(): + model = local_model() + key = ("local", base, model) + if key in seen: + continue + seen.add(key) + models.append({ + "id": model, + "label": f"{model} · local", + "source": "local", + "vision": True, + "base_url": base, + }) + try: + from src.endpoint_resolver import resolve_endpoint + + for prefix in ("utility", "default"): + try: + url, model, _headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model: + key = ("hosted", model) + if key not in seen: + seen.add(key) + models.append({ + "id": model, + "label": model, + "source": "hosted", + "vision": None, + "base_url": url, + }) + break + except Exception: + pass + return models + + +# --------------------------------------------------------------------------- +# Reachability probe (best effort, non-blocking). +# --------------------------------------------------------------------------- + +_PROBE: Dict[str, bool] = {} + + +def probe_models(base: str, timeout: float = 3.0) -> bool: + try: + with httpx.Client(timeout=timeout) as client: + resp = client.get(str(base).rstrip("/") + "/models") + return resp.status_code == 200 + except Exception: + return False + + +def _probe_worker() -> None: + for base in local_bases(): + ok = probe_models(base) + _PROBE[base] = ok + logger.info("editor agent local base %s reachable=%s", base, ok) + cap = caption_base() + if cap: + _PROBE[cap] = probe_models(cap) + logger.info("editor agent caption base %s reachable=%s", cap, _PROBE[cap]) + + +def startup_probe() -> None: + """Kick off a background reachability probe (never blocks startup).""" + try: + threading.Thread(target=_probe_worker, name="editor-agent-probe", daemon=True).start() + except Exception: + logger.debug("editor agent probe thread failed", exc_info=True) + + +def probe_status() -> Dict[str, bool]: + return dict(_PROBE) diff --git a/src/editor_style_guide.py b/src/editor_style_guide.py new file mode 100644 index 000000000..0f03497ed --- /dev/null +++ b/src/editor_style_guide.py @@ -0,0 +1,1105 @@ +"""Single source of truth for photo-editor intent routing. + +Both the server-side planner (``routes/editor_agent_routes.py``) and the +browser fallback planner (``static/js/editor/nl-edit.js``) read this guide so +their behaviour cannot drift. It is served verbatim to the client at +``GET /api/editor/style-guide``. + +Calibrated strengths live here; this file is the only place to tune routing. +It is intentionally dependency-free so the route module can import it cheaply. +""" +from __future__ import annotations + +import json +import re +from collections import Counter +from typing import Any, Dict, List, Optional + +STYLE_GUIDE_VERSION = 8 + +# New editor ops (implemented client-side, static) with canonical params. The +# server op catalog must include these or a valid plan would be filtered out. +NEW_OPS: List[Dict[str, Any]] = [ + { + "op": "photo.autoTone", + "params": {}, + "help": "Camera-Raw-style auto tone/colour (per-channel levels, auto white " + "balance, exposure, contrast). The single preferred global lift.", + }, + { + "op": "retouch.skin", + "params": {"smooth": 8, "opacity": 0.35, "blemish": True, + "radius": 12, "threshold": 16}, + "help": "Gentle frequency-separation skin smoothing that keeps texture, over " + "the selection intersected with skin tones. Keep it subtle: smooth " + "<=14, opacity <=0.45; it never brightens the face.", + }, + { + "op": "retouch.blemish", + "params": {"radius": 4, "threshold": 16}, + "help": "Remove small blemishes inside the selection or skin region.", + }, + { + "op": "retouch.teeth", + "params": {"desat": 0.6, "bright": 16}, + "help": "Whiten teeth within the current selection (desaturate yellow, " + "lift brightness with a small feather).", + }, + { + "op": "selection.face", + "params": {"mode": "replace"}, + "help": "Select a face (SAM text 'face', upper-centre ellipse fallback).", + }, + { + "op": "selection.background", + "params": {"mode": "replace", "subject": "person", "fuzziness": 30}, + "help": "Select the background (SAM subject inverted, corner-colour " + "fallback). Subject-aware; prefer over a plain global blur.", + }, + { + "op": "fill.flood", + "params": {"color": "#ff0000", "x": 0, "y": 0, "tolerance": 32}, + "help": "Paint-bucket flood-fill a pixel layer from a point " + "{x,y,color,tolerance}. Use for 'fill ... with ' when there " + "is no active selection.", + }, + { + "op": "selection.fill", + "params": {"color": "#ff0000"}, + "help": "Fill the current selection with a colour on the active pixel layer " + "{color}. Use for 'fill the selection with '.", + }, + { + "op": "text.edit", + "params": {"content": "New text", "size": 96, "color": "#ff0000"}, + "help": "Edit an EXISTING text layer {id|index|name, content, font, size, " + "color, align, tracking, lineHeight}. Use for 'make the text " + "bigger/smaller' and 'make the title ' — never text.add.", + }, + { + "op": "layers.align", + "params": {"mode": "center"}, + "help": "Align/distribute the selected layers to the canvas {mode:left|center|" + "right|top|middle|bottom|distribute-h|distribute-v}.", + }, + { + "op": "layer.rotate", + "params": {"degrees": 90}, + "help": "Rotate the ACTIVE LAYER by 90/180/270 {degrees, id|index|name}. Use " + "unless the prompt names the whole image/canvas/photo.", + }, + { + "op": "layer.flip", + "params": {"axis": "h"}, + "help": "Flip the ACTIVE LAYER in place {axis:h|v, id|index|name}.", + }, + { + "op": "document.rotate", + "params": {"degrees": 90}, + "help": "Rotate the WHOLE document (all layers + canvas) by 90/180/270 " + "{degrees} — for 'rotate the whole image/canvas/photo'.", + }, + { + "op": "document.flip", + "params": {"axis": "h"}, + "help": "Flip the WHOLE document {axis:h|v} — for 'flip the canvas/whole " + "image'.", + }, + { + "op": "selection.copy", + "params": {}, + "help": "Copy (or cut, {cut:true}) the selection pixels to the internal " + "clipboard.", + }, + { + "op": "selection.paste", + "params": {}, + "help": "Paste the internal clipboard as a new layer {name}.", + }, + { + "op": "layers.flatten", + "params": {}, + "help": "Flatten all layers into a single layer.", + }, + { + "op": "layers.mergeDown", + "params": {}, + "help": "Merge the active layer down into the layer below.", + }, +] + +NEW_OP_NAMES: List[str] = [op["op"] for op in NEW_OPS] + + +def _adj(kind: str, params: Dict[str, Any]) -> Dict[str, Any]: + return {"op": "adjustments.apply", "params": {"type": kind, "params": params}} + + +def _filter(fid: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + return {"op": "filters.apply", "params": {"id": fid, **(params or {})}} + + +# Ordered intent routes. The first route whose keyword appears wins. +INTENT_ROUTES: List[Dict[str, Any]] = [ + { + "id": "teeth", + "keywords": ["teeth", "whiten my smile", "whiter smile", "smile looks"], + "steps": [ + {"op": "selection.face", "params": {"mode": "replace"}}, + {"op": "retouch.teeth", "params": {"desat": 0.6, "bright": 16}}, + ], + "reply": "Selecting the face and whitening teeth.", + }, + { + "id": "portrait_retouch", + "keywords": ["skin", "complexion", "blemish", "acne", "pores", "retouch", + "look better", "look my best", "make me look", "beautify", + "flawless", "smooth my face", "clear up", "clean up my face", + "portrait", "selfie"], + # Do NOT lead with a whole-image photo.autoTone: lightening the whole frame + # washes out the face (the client suppresses autoTone for portrait intents + # unless the histogram shows real under/over-exposure). Go straight to a + # gentle skin retouch; keep it subtle and never brighten the face. + "min_steps": 3, + "steps": [ + {"op": "selection.face", "params": {"mode": "replace"}}, + {"op": "retouch.skin", "params": {"smooth": 8, "opacity": 0.35, + "blemish": True, "radius": 12, + "threshold": 16}}, + {"op": "selection.deselect", "params": {}}, + ], + "reply": "Selecting the face and gently retouching the skin.", + }, + { + "id": "restore", + "keywords": ["restore", "restoration", "old photo", "old photograph", + "old picture", "faded photo", "faded picture", "faded", + "recover this photo", "bring back this photo", + "photo restoration"], + # Never a large exposure lift: recover contrast + colour from a faded + # baseline, then a slight sharpen. + "min_steps": 4, + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("brightness-contrast", {"brightness": 0, "contrast": 18}), + _adj("white-balance", {"temperature": 0, "tint": 5}), + _adj("vibrance", {"vibrance": 18}), + _filter("sharpen"), + ], + "reply": "Restoring contrast and colour, then a light sharpen.", + }, + { + "id": "background", + "keywords": ["background"], + "steps": [ + {"op": "selection.background", + "params": {"mode": "replace", "subject": "person", "fuzziness": 30}}, + {"op": "filters.applyMasked", "params": {"id": "blur-gaussian"}}, + ], + "reply": "Selecting the background and softening it.", + }, + { + "id": "lighting", + # Low-key lighting: one auto-tone pass (the preferred global lift) plus + # at most one shadow-lifting, highlight-protecting adjustment. Both are + # sanctioned together; never a stack of brightness/exposure lifts. + "keywords": ["lighting", "exposure", "too dark", "underexposed", + "underexpose", "backlit", "back-lit", "brighten", "brighter", + "lift the shadows", "dark photo", "dark image"], + "min_steps": 2, + "steps": [ + {"op": "photo.autoTone", "params": {}}, + {"op": "adjustments.apply", + "params": {"type": "shadows-highlights", + "params": {"shadows": 30, "highlights": -15}}}, + ], + "reply": "Auto tone (levels and white balance), then one shadow lift " + "that protects the highlights.", + }, + { + "id": "cool", + "keywords": ["moodier", "moody", "cooler", "cool it", "cinematic", "teal", + "blue tone", "cold", "night mood", "colder"], + "steps": [_adj("white-balance", {"temperature": -25})], + "reply": "Cooling the white balance (negative temperature).", + }, + { + "id": "warm", + "keywords": ["warmer", "warm it", "golden", "summer", "sunset", + "golden hour", "warmth"], + "steps": [_adj("white-balance", {"temperature": 25})], + "reply": "Warming the white balance (positive temperature).", + }, + { + "id": "black_white", + "keywords": ["black and white", "black & white", "grayscale", "greyscale", + "monochrome", "b&w"], + "steps": [_adj("black-white", {})], + "reply": "Converting to black and white.", + }, + { + "id": "polish", + # Generic "make it look good" intents: a decisive multi-step grade with + # visible strength (a single timid adjustment scores as imperceptible). + "keywords": ["pretty", "beautiful", "make it look good", "look good", + "professional", "professionally", "editorial", "magazine", + "glam", "glamour", "pop", "punchy", "striking", "vibrant", + "film", "film look", "cinematic look", "instagram", + "social media", "elevate", "polish"], + "min_steps": 3, + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("vibrance", {"vibrance": 25}), + _filter("clarity", {"amount": 22, "radius": 30}), + _filter("vignette", {"amount": 22, "feather": 0.6}), + ], + "reply": "Balancing tone, adding vibrance and local contrast, with a light vignette.", + }, + { + "id": "vivid", + "keywords": ["vivid", "dramatic", "saturate", "saturation", + "colourful", "colorful"], + "steps": [_adj("vibrance", {"vibrance": 35})], + "reply": "Boosting vibrance.", + }, + { + "id": "sharpen", + "keywords": ["sharpen", "crisper", "crisp", "more detail", "sharper"], + "steps": [_filter("sharpen")], + "reply": "Sharpening.", + }, +] + +# A prompt that only asks for a vague "improvement" with no concrete defect and +# no matching intent cannot be planned deterministically -> ask the user. +_AMBIGUOUS_PATTERNS = [ + "clean this up", "clean it up", "fix it", "fix this", "make it better", + "make this better", "improve", "touch up", "touch it up", "do something", + "enhance", "make it nice", "polish", +] +_AMBIGUOUS_OPTIONS = [ + {"label": "Auto tone & color", + "description": "photo.autoTone: balanced levels, white balance, contrast."}, + {"label": "Brighten (lift shadows)", + "description": "shadows-highlights: brighten without blowing highlights."}, + {"label": "Cooler / cinematic", + "description": "white-balance negative: cooler, moodier look."}, +] +_DEFECT_HINTS = [ + "skin", "teeth", "background", "light", "dark", "colour", "color", "warm", + "cool", "blur", "sharpen", "contrast", "bright", "shadow", "face", + "blemish", "noise", "grain", "crop", "vibrance", "saturat", "black and white", + "grayscale", "portrait", "moody", "exposure", "tone", +] + +# Low-key safety: at most ONE global lift; prefer shadows/highlights and keep +# highlights protected. Global-lift adjustment types are counted together. +GLOBAL_LIFT_TYPES = ("autoTone", "brightness-contrast", "exposure", "levels", + "shadows-highlights") +_SAFE_LOWKEY_LIFT = { + "op": "adjustments.apply", + "params": {"type": "shadows-highlights", + "params": {"shadows": 30, "highlights": -15}}, +} + + +def match(prompt: str) -> Optional[Dict[str, Any]]: + """Return the first matching intent route for a prompt, or None.""" + text = str(prompt or "").lower() + for route in INTENT_ROUTES: + if any(keyword in text for keyword in route["keywords"]): + return route + return None + + +def is_ambiguous(prompt: str) -> bool: + text = str(prompt or "").lower() + if match(text): + return False + if any(hint in text for hint in _DEFECT_HINTS): + return False + return any(pattern in text for pattern in _AMBIGUOUS_PATTERNS) + + +def ask_user_plan() -> Dict[str, Any]: + question = "What would you like me to do with this photo?" + return { + "reply": question, + "planner": "fallback", + "steps": [], + "ask_user": { + "question": question, + "options": [dict(o) for o in _AMBIGUOUS_OPTIONS], + "multi": False, + }, + } + + +def has_concrete_intent(prompt: str) -> bool: + """True when a prompt maps to a concrete edit (a recipe, look, recolour, + structural op) or names a concrete defect. Used to decide whether a plan made + only of generic whole-image tone ops should be trusted or replaced by an + ask_user prompt. + """ + text = str(prompt or "").lower() + if match(text) or look_plan(text) or recolor_plan(text) or structural_plan(text): + return True + return any(hint in text for hint in _DEFECT_HINTS) + + +def fallback_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan from the guide, or None when nothing matches.""" + route = match(prompt) + if route: + return { + "reply": route["reply"], + "planner": "fallback", + "steps": enforce_guards([dict(step) for step in route["steps"]]), + } + if is_ambiguous(prompt): + return ask_user_plan() + return None + + +# ── Structural op routing ────────────────────────────────────────────── +# Deterministic plans for structural editor ops the sampling model reliably +# substitutes with a generic adjustments/transform step. Mirrors the browser +# fallback (static/js/editor/nl-edit.js localPlan) so the server and client +# agree; the server path must emit these or a wrong model plan suppresses the +# browser fallback. +_NAMED_COLORS = { + "red": "#ff0000", "blue": "#0000ff", "green": "#00a000", "yellow": "#ffff00", + "orange": "#ff8000", "purple": "#8000ff", "pink": "#ff69b4", "black": "#000000", + "white": "#ffffff", "grey": "#808080", "gray": "#808080", "cyan": "#00ffff", + "magenta": "#ff00ff", "brown": "#8b4513", "teal": "#008080", +} +_COLOR_TOKEN = (r"(red|blue|green|yellow|orange|purple|pink|black|white|grey|" + r"gray|cyan|magenta|brown|teal|#[0-9a-fA-F]{3,6})") + + +def color_for_name(name: str) -> Optional[str]: + n = str(name or "").strip().lower() + if re.match(r"^#[0-9a-f]{3,6}$", n): + return n + return _NAMED_COLORS.get(n) + + +# ── Named reference looks + targeted recolour ────────────────────────── +# Server-only deterministic plans (reached via routes._plan_special -> +# structural_plan). Deliberately NOT added to INTENT_ROUTES: the browser +# client consumes `intents` and cannot run an ai_edit_image step, so strong +# looks must stay on the server path. +_LOOK_KEEP_CLAUSE = " Keep the main subject and composition exactly the same." + +LOOKS = [ + { + "id": "wes_anderson", + "keywords": ["wes anderson"], + "mode": "recipe", + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("white-balance", {"temperature": 10, "tint": 6}), + _adj("vibrance", {"vibrance": 18}), + _adj("brightness-contrast", {"brightness": 6, "contrast": -10}), + ], + "reply": "Gave it a Wes Anderson look: pastel palette, warm highlights and soft contrast.", + }, + { + "id": "film_noir", + "keywords": ["film noir"], + "mode": "recipe", + "steps": [ + _adj("black-white", {}), + _adj("brightness-contrast", {"contrast": 38}), + _adj("shadows-highlights", {"shadows": -22, "highlights": 18}), + _filter("vignette", {"amount": 45, "feather": 0.5}), + ], + "reply": "Gave it a film-noir look: high-contrast black and white with deep shadows.", + }, + { + "id": "vintage_kodachrome", + "keywords": ["kodachrome", "vintage film"], + "mode": "recipe", + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("white-balance", {"temperature": 12, "tint": 8}), + _adj("vibrance", {"vibrance": 22}), + _adj("brightness-contrast", {"contrast": 14}), + _filter("vignette", {"amount": 12, "feather": 0.7}), + ], + "reply": "Gave it a vintage Kodachrome look: warm saturated colour and lifted contrast.", + }, + { + "id": "cyberpunk", + "keywords": ["cyberpunk"], + "mode": "instruction", + "prompt": "Restyle the photo as a cyberpunk scene: vivid neon magenta and cyan " + "lighting, wet reflective surfaces, deep shadows and a futuristic " + "atmosphere." + _LOOK_KEEP_CLAUSE, + "reply": "Restyling it as a cyberpunk scene with the image model.", + }, + { + "id": "studio_ghibli", + "keywords": ["ghibli", "studio ghibli"], + "mode": "instruction", + "prompt": "Restyle the photo as a soft hand-painted anime scene in a " + "Studio-Ghibli-inspired pastel style: luminous pastel palette, " + "gentle painterly shading, clean edges and warm sunlight." + + _LOOK_KEEP_CLAUSE, + "reply": "Restyling it as a Studio-Ghibli-inspired pastel scene with the image model.", + }, +] + + +def look_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan for a named reference look, or None. + + Recipe looks (Wes Anderson, film noir, vintage Kodachrome) are pure + adjustment/filter stacks; strong looks (cyberpunk, Studio Ghibli) use + ai_edit_image instruction mode. + """ + p = str(prompt or "").lower() + for look in LOOKS: + if any(k in p for k in look["keywords"]): + if look["mode"] == "instruction": + steps = [{"op": "ai_edit_image", + "params": {"op": "generative", "mode": "instruction", + "prompt": look["prompt"], "steps": 8}}] + else: + steps = [dict(s) for s in look["steps"]] + return {"reply": look["reply"], "planner": "deterministic", "steps": steps} + return None + + +_RECOLOR_ACTION_RE = re.compile( + r"\brecolou?r\b" + r"|\bmake\s+only\b" + r"|\bmake\s+the\b[^.]{0,30}?\b" + _COLOR_TOKEN + r"\b" + r"|\b(?:change|turn|convert|paint|colou?r)\b[^.]{0,40}?\b" + _COLOR_TOKEN + r"\b", + re.I, +) +_RECOLOR_EXCLUDE_RE = re.compile( + r"\b(?:background|backdrop|blur|black\s+and\s+white|grayscale|greyscale|" + r"monochrome|b&w|text|title|font|caption|label|logo|watermark|drop\s+shadow|" + r"generative)\b", + re.I, +) + + +_RECOLOR_TRAILING_ADJ = re.compile( + r"\s+\b(?:bright|brighter|vivid|vibrant|dark|darker|light|lighter|deep|" + r"pale|more|less|very|really|slightly|totally|completely)\b\s*$", + re.I, +) + + +def _hex_rgb01(hex_color: str): + h = str(hex_color or "").lstrip("#") + if len(h) == 3: + h = "".join(ch * 2 for ch in h) + try: + return tuple(int(h[i:i + 2], 16) / 255.0 for i in (0, 2, 4)) + except Exception: + return None + + +def _hue_degrees(hex_color: str) -> Optional[float]: + rgb = _hex_rgb01(hex_color) + if rgb is None: + return None + r, g, b = rgb + mx, mn = max(rgb), min(rgb) + if mx == mn: + return 0.0 + d = mx - mn + if mx == r: + h = ((g - b) / d) % 6 + elif mx == g: + h = (b - r) / d + 2 + else: + h = (r - g) / d + 4 + return h * 60.0 + + +def _hue_delta(src_hex: str, tgt_hex: str) -> int: + sh = _hue_degrees(src_hex) + th = _hue_degrees(tgt_hex) + if sh is None or th is None: + return 0 + return int(round((th - sh + 180.0) % 360.0 - 180.0)) + + +def _is_chromatic(hex_color: str) -> bool: + """True when a pure hue rotation can plausibly reach this colour. + + White / black / grey targets have (near) zero saturation and cannot be + reached by rotating hue, so they go through the image model instead. + """ + rgb = _hex_rgb01(hex_color) + if rgb is None: + return False + mx, mn = max(rgb), min(rgb) + return (mx - mn) >= 0.15 and mx >= 0.12 + + +def _recolor_object_phrase(prompt: str, target_word: str) -> str: + """Best-effort object description for the mask ("red car" in 'the red car').""" + m = re.search( + r"\b(?:make|recolou?r|change|turn|convert|paint|colou?r)\b\s+(?:only\s+)?(.+)", + prompt, re.I, + ) + phrase = (m.group(1) if m else prompt).strip() + if target_word: + phrase = re.sub(r"\b" + re.escape(target_word) + r"\b", " ", phrase, + count=1, flags=re.I) + phrase = re.sub(r"\s+", " ", phrase).strip(" .,!?") + phrase = re.sub(r"^(?:to|into|the|a|an)\s+", "", phrase, flags=re.I).strip() + phrase = re.sub(r"\s+(?:to|into)$", "", phrase, flags=re.I).strip() + while phrase and _RECOLOR_TRAILING_ADJ.search(phrase): + phrase = _RECOLOR_TRAILING_ADJ.sub("", phrase).strip() + return phrase or "the subject" + + +def recolor_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan for a targeted recolour, or None. + + 'make only the red car blue' -> select the OBJECT by description + (``selection.samText`` "red car"; falling back to a colour-range selection + via ``fallbackColor`` when the mask is empty) and then shift the MASKED + HUE toward the target with ``adjustments.applyMasked`` (preserving shading, + highlights and reflections). Never a flat ``selection.fill`` and never a + whole-image ``adjustments.apply`` tint. When the target cannot be reached by + a hue rotation (white/black/grey) or no source colour is known, the masked + region is recoloured with ``ai_edit_image`` instruction mode instead. + """ + p = str(prompt or "") + if not _RECOLOR_ACTION_RE.search(p): + return None + if _RECOLOR_EXCLUDE_RE.search(p): + return None + colors = [m.group(1).lower() for m in + re.finditer(r"\b" + _COLOR_TOKEN + r"\b", p)] + colors = [c for c in colors if color_for_name(c)] + if not colors: + return None + target = colors[-1] + tgt_hex = color_for_name(target) + source = colors[0] if len(colors) >= 2 and colors[0] != target else None + src_hex = color_for_name(source) if source else None + phrase = _recolor_object_phrase(p, target) + select = { + "op": "selection.samText", + "params": {"text": phrase}, + } + if src_hex: + select["params"]["fallbackColor"] = src_hex + select["params"]["fallbackFuzziness"] = 40 + if source and src_hex and _is_chromatic(tgt_hex): + delta = _hue_delta(src_hex, tgt_hex) + src_hue = int(round(_hue_degrees(src_hex) or 0)) + return { + "reply": "Selected the %s and shifted its hue toward %s " + "(shading kept)." % (phrase, target), + "planner": "deterministic", + "steps": [ + select, + {"op": "adjustments.applyMasked", + "params": {"hue": delta, "saturation": 0, + "mode": "hue-saturation", + "sourceHue": src_hue, + "source": src_hex, "target": tgt_hex}}, + ], + } + instruction = ("Recolour the %s to %s, preserving its shading, highlights " + "and reflections." % (phrase, target)) + return { + "reply": "Recolouring the %s with the image model." % phrase, + "planner": "deterministic", + "steps": [ + select, + {"op": "ai_edit_image", + "params": {"op": "inpaint", "mode": "masked", + "prompt": instruction, "steps": 8}}, + ], + } + + +def structural_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan for structural editor ops, or None. + + Covers fill/flood, text.edit, colour-overlay layer style, layer align, + layer/document rotate+flip, selection copy/paste, flatten and merge-down. + Also dispatches named reference looks and targeted recolours. + Returns ``{"reply", "planner": "deterministic", "steps": [...]}``. + """ + look = look_plan(prompt) + if look is not None: + return look + recolor = recolor_plan(prompt) + if recolor is not None: + return recolor + p = str(prompt or "").lower() + steps: List[Dict[str, Any]] = [] + labels: List[str] = [] + + fm = (re.search(r"\bfill\b[^.]*?\bwith\s+" + _COLOR_TOKEN + r"\b", p) + or re.search(r"\bfill\b[^.]*?\b" + _COLOR_TOKEN + r"\b", p)) + if fm: + color = color_for_name(fm.group(1)) + if color: + if re.search(r"\bselection\b|\bselected\b", p): + steps.append({"op": "selection.fill", "params": {"color": color}}) + labels.append("filled the selection") + else: + steps.append({"op": "fill.flood", + "params": {"color": color, "x": 0, "y": 0}}) + labels.append("flood-filled the layer") + + if re.search(r"\b(text|title|caption|font|label)\b", p) and not re.search( + r"\b(add|insert|create|write|put|place)\b", p): + tparams: Dict[str, Any] = {} + if re.search(r"\b(bigger|larger|increase)\b", p): + tparams["size"] = 96 + elif re.search(r"\b(smaller|reduce|decrease)\b", p): + tparams["size"] = 28 + cm = re.search(r"\b" + _COLOR_TOKEN + r"\b", p) + if cm: + c = color_for_name(cm.group(1)) + if c: + tparams["color"] = c + if tparams: + steps.append({"op": "text.edit", "params": tparams}) + labels.append("updated the text layer") + + if re.search(r"\bcolo?ur?\s*overlay\b", p): + steps.append({"op": "layers.addStyle", + "params": {"type": "color-overlay", "params": {}}}) + labels.append("added a colour-overlay layer style") + + am = re.search( + r"\balign\b[^.]*?\b(centre|center|left|right|top|middle|bottom)\b", p) + if am: + mode = "center" if am.group(1) in ("centre", "center") else am.group(1) + steps.append({"op": "layers.align", "params": {"mode": mode}}) + labels.append("aligned to the %s" % mode) + + whole = bool(re.search( + r"\b(?:whole|entire|full)\s+(?:image|canvas|photo|picture|document)\b", p)) \ + or bool(re.search(r"\b(?:canvas|document)\b", p) + and re.search(r"\b(?:rotate|flip|mirror|turn)\b", p)) + if re.search(r"\b(?:flip|mirror)\b", p): + axis = "v" if re.search(r"\b(?:vertical(?:ly)?|v)\b", p) else "h" + if whole: + steps.append({"op": "document.flip", "params": {"axis": axis}}) + labels.append("flipped the whole image") + else: + steps.append({"op": "layer.flip", "params": {"axis": axis}}) + labels.append("flipped the layer") + # Rotation needs an explicit angle/quarter-turn or the verb "rotate"; a bare + # "turn ... into