Integrate updated editor backend review

This commit is contained in:
pewdiepie-archdaemon
2026-10-26 16:36:59 +00:00
45 changed files with 9690 additions and 88 deletions
+83 -1
View File
@@ -178,6 +178,14 @@ app.add_middleware(SecurityHeadersMiddleware)
import asyncio as _asyncio
from starlette.middleware.base import BaseHTTPMiddleware as _BaseHTTPMiddleware
from starlette.responses import JSONResponse as _JSONResponse
import anyio as _anyio
from starlette.requests import ClientDisconnect as _ClientDisconnect
# SSE endpoints (e.g. /api/editor/bridge/events) end/abort mid-stream when the
# browser disconnects; Starlette's BaseHTTPMiddleware surfaces that as
# anyio.EndOfStream (and sometimes ClientDisconnect). Those are client-side
# events, not server errors, so we map them to a 499 instead of a 500 traceback.
_STREAM_CLIENT_EXC = (_anyio.EndOfStream, _ClientDisconnect)
REQUEST_HARD_TIMEOUT = float(os.getenv("REQUEST_HARD_TIMEOUT", "45"))
_TIMEOUT_EXEMPT_PREFIXES = (
@@ -190,6 +198,7 @@ _TIMEOUT_EXEMPT_PREFIXES = (
"/api/cookbook/setup", # remote pacman/apt installs
"/api/upload", # large files
"/api/image", # diffusion proxies (inpaint/harmonize/upscale/etc.) — own 120s httpx timeout
"/api/editor", # editor agent/bridge: generative steps can take 30-110s (SAM + local diffusion)
"/api/memory/audit", # retains own 120s LLM inactivity timeout
)
@@ -225,15 +234,43 @@ class _InteractiveActivityMiddleware(_BaseHTTPMiddleware):
return await call_next(request)
# Heavy image ops that can run 30-150 s; a deploy restart must not cut these off.
_HEAVY_IMAGE_OP_PREFIXES = (
"/api/image/inpaint",
"/api/image/replace-background",
"/api/image/outpaint",
"/api/image/extend",
"/api/image/remove-bg",
"/api/image/mask",
"/api/image/upscale-local",
"/api/image/denoise",
"/api/image/enhance-face",
"/api/gallery/ai-upscale",
"/api/gallery/style-transfer",
"/api/editor/agent",
)
class _SlowRequestLogMiddleware(_BaseHTTPMiddleware):
async def dispatch(self, request, call_next):
from src import activity
path = request.url.path or ""
heavy = any(path.startswith(p) for p in _HEAVY_IMAGE_OP_PREFIXES)
if heavy:
activity.begin_image_op(path)
start = time.perf_counter()
status = 500
try:
response = await call_next(request)
status = getattr(response, "status_code", 0) or 0
return response
except _STREAM_CLIENT_EXC:
# Client disconnected / SSE stream ended — not a server error.
status = 499
return _JSONResponse({"detail": "client disconnected"}, status_code=499)
finally:
if heavy:
activity.end_image_op(path)
elapsed = time.perf_counter() - start
try:
threshold = float(os.getenv("ODYSSEUS_SLOW_REQUEST_LOG_SECONDS", "0.75") or "0.75")
@@ -822,6 +859,10 @@ app.include_router(setup_gallery_routes())
from routes.editor_draft_routes import setup_editor_draft_routes
app.include_router(setup_editor_draft_routes())
# Natural-language editor planning (planner wired to the configured chat model)
from routes.editor_agent_routes import setup_editor_agent_routes
app.include_router(setup_editor_agent_routes())
# Scheduled tasks + event bus
from src.task_scheduler import TaskScheduler
task_scheduler = TaskScheduler(session_manager)
@@ -1000,6 +1041,16 @@ async def get_version():
"source_commit": APP_SOURCE_COMMIT,
}
@app.get("/api/busy")
async def get_busy():
"""Report in-flight heavy work so a deploy can wait instead of dropping it."""
from src import activity
return {
"busy_image_ops": activity.heavy_image_ops_active(),
**activity.activity(),
}
@app.get("/api/health")
async def health_check() -> Dict[str, str]:
return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()}
@@ -1438,4 +1489,35 @@ if __name__ == "__main__":
bind_host = os.getenv("APP_BIND", "127.0.0.1")
bind_port = int(os.getenv("APP_PORT", "7011"))
uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")
# Give in-flight requests time to finish on SIGTERM instead of being cut
# off mid-flight. Diffusion edits run 60-120 s; outpaint/extend can run up
# to ~330 s (provider cap 900 s), so default the graceful window to 900 s.
# systemd's TimeoutStopSec must exceed this (see the graceful-stop drop-in).
#
# We hook uvicorn's signal handler so a SIGTERM also flips src.shutdown:
# SSE / streaming generators check it and end promptly, so an open browser
# stream cannot stall a restart for the whole window. The long window stays
# a safety net for genuine in-flight image ops.
from src import shutdown as _shutdown
config = uvicorn.Config(
app,
host=bind_host,
port=bind_port,
log_level="info",
timeout_graceful_shutdown=int(os.getenv("ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT", "900") or 900),
)
server = uvicorn.Server(config)
# uvicorn's capture_signals() wires SIGINT/SIGTERM to self.handle_exit, so
# wrapping that method lets us set the shutdown flag before the normal exit
# path runs (this uvicorn version has no install_signal_handlers kwarg).
_uvicorn_handle_exit = server.handle_exit
def _handle_shutdown_signal(signum, frame):
_shutdown.request_shutdown()
_uvicorn_handle_exit(signum, frame)
server.handle_exit = _handle_shutdown_signal
server.run()
+201
View File
@@ -0,0 +1,201 @@
# Editor Agent API
How an agent (LLM or script) drives the PhotoCraft editor. There are three layers:
1. **Control API** — `window.geEditorControl`, a stable in-page command surface.
2. **NL planner** — `window.geEditorAI`, turns a prompt into control-API steps (LLM or basic mode).
3. **Bridge** — server-mediated SSE channel so a *server-side* agent can drive an **open** editor.
All three are owner-scoped by the app session; nothing is exposed to unauthenticated users.
---
## 1. Control API (`window.geEditorControl`)
Installed when the editor opens. Version `EDITOR_CONTROL_VERSION`.
```js
await window.geEditorControl.invoke(op, params) // { ok, op, result } | { ok:false, op, error }
await window.geEditorControl.run([{op, params}]) // sequential, stops on first failure
window.geEditorControl.list() // [{op, description}]
```
`invoke` is async. Every op that mutates the document pushes **one** undo entry via
`_saveState`, so a single op is one undo step.
### Op catalog
| Op | Params | Notes |
| --- | --- | --- |
| `document.info` | – | size, layers, mode |
| `document.importImage` | `{dataUrl,name,raster}` | load a raster image as a new layer |
| `layers.list` | – | top-most first |
| `layers.select` | `{id\|index\|name}` | index is top-most-first |
| `layers.addRaster` | – | new empty pixel layer |
| `layers.addAdjustment` | `{type,params}` | adjustment layer |
| `layers.duplicate` / `layers.delete` | – | on the active/selected layer(s) |
| `layers.setOpacity` | `{id?,opacity 0-1}` | |
| `layers.setBlend` | `{id?,blendMode\|mode}` | |
| `layers.group` / `layers.ungroup` | – | |
| `selection.selectAll` / `deselect` / `invert` | – | |
| `selection.rectangle` / `selection.ellipse` | `{x,y,w,h}` | document space |
| `selection.subject` | `{label}` | SAM text grounding (falls back to a centred rect) |
| `selection.face` | – | SAM "face" (falls back to an upper-centre ellipse) |
| `selection.sky` | – | SAM "sky" (falls back to a top-band colour range) |
| `selection.background` | `{fuzziness}` | border colour union |
| `selection.samText` | `{text}` | SAM text grounding only (throws without SAM) |
| `selection.colorRange` | `{color,fuzziness}` | |
| `selection.hasMask` | – | `{active, space}` |
| `adjustments.apply` | `{type,params}` | adjustment layer; camelCase types accepted |
| `adjustments.applyMasked` | `{brightness,contrast,saturation,warmth,invert}` | pixel pass, masked if a selection exists else whole layer |
| `filters.apply` | `{id}` | opens the slider dialog and applies |
| `filters.applyMasked` | `{id,radius,amount,invert}` | `id`: blur-gaussian, sharpen, unsharp-mask |
| `transform.start` / `transform.apply` / `transform.cancel` | – | |
| `text.add` | `{text,x,y,color,size,font,align}` | |
| `history.undo` / `history.redo` | – | |
| `history.mark` | – | returns current undo depth (for grouping) |
| `history.collapse` | `{mark,label}` | collapses entries above `mark` into one |
| `export.png` | – | `{dataUrl}` of the flattened document |
| `export.gallery` | – | saves a copy back to the gallery |
| `screenshot` | – | `{dataUrl,width,height}` of the composited canvas |
Param conventions the planner must respect:
- `adjustments.applyMasked`: `brightness`/`contrast`/`warmth` are **-100..100** (0 neutral);
`saturation` is a **multiplier** (1 neutral; values >3 are treated as percentages).
- `adjustments.apply` `type` aliases are normalised (`blackAndWhite` -> `black-white`).
- `color-balance` params are `{shadows:{r,g,b}, midtones:{r,g,b}, highlights:{r,g,b}}`;
a bare number is coerced to `{r:n,g:n,b:n}`.
- A masked op with **no** active selection degrades to a whole-layer edit (does not throw).
---
## 2. NL planner (`window.geEditorAI`)
```js
const res = await window.geEditorAI.execute('blur the background behind the person', { onStep });
// exec: window.geEditorAI.execute('blur the background behind the person', { onStep });
// res = { ok, prompt, reply, planner, basicMode, steps, trace, failed }
```
Flow (`static/js/editor/nl-edit.js`):
1. Read document state + a **before** screenshot.
2. Ask the **server planner** (`POST /api/editor/plan`) with the prompt, state and
screenshot. The client passes its strong guidance via `state.planner_guidance`.
If the route is unavailable it falls back to `localPlan` (regex — labelled
**"basic mode"** in the UI).
3. `sanitizeSteps()`: drop no-op/export steps and, if a masked op has no preceding
`selection.*`, insert the region implied by the prompt (face/sky/background/subject).
4. `history.mark`, then run each step, screenshotting after each one.
5. **Self-check** (server plans only, max 3 rounds): send the after screenshot back in
`mode:"check"` with the original intent. If not satisfied, **revert** the previous
attempt (undo its history entries) and apply the model's **full corrected** plan.
6. `history.collapse` — the whole request becomes a single undoable step.
UI: the Quick Edit box routes multi-clause / "and ..." prompts to this executor and
streams `Step i/n: <op>` into its status line; a `(basic mode)` suffix marks the offline planner.
### Server route
`POST /api/editor/plan` (HARNESS, `routes/editor_agent_routes.py`)
```
{ prompt, state, mode?: "plan"|"check", screenshot?: "data:image/..." }
-> { reply, planner, steps: [{op, params}], done? }
```
Owner-scoped, authenticated. Uses the configured default/utility model with vision.
If the model call is unavailable it returns a deterministic `planner:"fallback"` plan.
---
## 3. Bridge (server agent drives an open editor)
`static/js/editor/editor-bridge.js` subscribes the open editor to the server queue.
Enabled automatically when the editor opens (`window.__geEditorBridge`).
- Subscribe: `GET /api/editor/bridge/events?session=<id>` — SSE, `data: {id,op,params}` (keepalive every 15s).
- The page runs the op via the control API and replies:
`POST /api/editor/bridge/result {id,ok,result,error,screenshot,ms}`.
- Enqueue a command (server side): `POST /api/editor/bridge/command {session, op|steps, wait, timeout}`.
With `wait:true` the call blocks and returns the editor's result.
Round-trip example (authenticated; `<session>` = `window.__geEditorBridge.session`):
```bash
curl -s -X POST http://127.0.0.1:7011/api/editor/bridge/command \
-H 'Content-Type: application/json' -b "odysseus_session=$TOKEN" \
-d '{"session":"default","op":"layers.addRaster","wait":true,"timeout":8000}'
# -> {"id":"...","ok":true,"result":{...},"screenshot":"data:image/png;base64,..."}
```
Session scoping is `owner:session`, so an agent can only drive editors owned by the
same user. The bridge is a **browser-only** flow: it requires an editor page to be open.
---
## 4. SAM dependency
`selection.subject` / `selection.face` / `selection.sky` / `selection.samText` use the
server SAM mask tool (`POST /api/image/mask`). When the SAM tools are **not installed**
the editor raises *"Object mask tools are not installed"* and the control op falls back
to a **coarse approximation** (centred rect / upper-centre ellipse / top-band colour range).
On a plain background the approximation is acceptable; on real photos it leaks, producing
visible rectangular/elliptical artifacts. Install the Cookbook SAM mask dependencies for
true subject/face/sky segmentation.
`selection.colorRange` and `selection.background` never need SAM (pure colour matching),
but they are also coarse on busy scenes.
---
## 5. Two ways for an agent to use this — pick one
### Option A — built-in tool (in-process)
Register an editor-control tool inside the app (same process as the model) that calls the
bridge/control API directly. Uses the app's normal login; nothing new is exposed. This is
the rejected stdio-MCP design's replacement: a server-side tool that talks to the bridge
needs either an internal token (auth-bypass) or to run in-process. **Decision pending (user).**
### Option B — browser-only flow (works today)
No new server tool. An external agent drives an open editor purely through the bridge
endpoints using the logged-in owner's session. Restricted to the editor the user has open.
This is the flow demonstrated above and requires **no** auth-bypass.
A stdio `photo_editor` MCP server was prototyped and **rejected** because the built-in MCP
subprocess has no internal token; wiring it would require expanding an auth bypass.
The two viable options above are left for the user to choose.
---
## 6. Known limitations / honest grading
Live run against real 7011 with the server LLM planner (7 prompts): all 7 executed
(`planner:"llm"`). Graded by eye:
- PASS: black & white portrait; warm cinematic; sharper + more vivid; text title.
- FAIL (region-masked, SAM-bound): "dramatic sky" (global over-saturation instead of a
dramatic, masked sky), "blur the background" (coarse mask leaks; visible rectangle),
"brighten the face" (coarse ellipse/rect artifact).
Whole-image edits are reliable; subject/face/sky region edits need SAM. The self-check
loop works (revert-and-replan) but the region cases spend all 3 iterations.
## 7. Theming (control kit)
The editor's interactive controls (ranges, number fields, selects, checkboxes) share a
single themed kit: `static/js/editor/control-kit.js` adds `.ge-ctl-*` classes scoped to
the editor roots (`#gallery-editor-container`, `.ge-adj-popup`, `.ge-filter-overlay`,
`.ge-canvas-size-overlay`, `.ge-float-window`, `.ge-transform-popup`), and
`static/css/13-image-editor.css` (section "Editor control kit") styles them from the app
design tokens (`static/css/00-tokens.css`) to match the app sliders/selects. It keeps
range fills in sync and attaches a **scrub-on-label** affordance to numeric fields
(drag the field's label to change the value; arrow keys work, Shift = x10).
**The PhotoCraft editor is dark-only.** `:root.light` flips the global tokens, but
`13-image-editor.css` has no `.light` handling and the editor relies on dark surfaces,
so light mode renders a garbled mix. Unlike the rest of Odysseus, the editor follows
Photoshop and is not offered in a light theme.
File diff suppressed because it is too large Load Diff
+84 -1
View File
@@ -43,6 +43,7 @@ from starlette.concurrency import run_in_threadpool
from fastapi import APIRouter, Query, UploadFile, File, BackgroundTasks, HTTPException, Depends, Request
from fastapi.responses import FileResponse, StreamingResponse
from src.constants import DATA_DIR
from src import shutdown as _shutdown
from src.path_confinement import confine
from src.llm_core import llm_call_async
@@ -1834,13 +1835,55 @@ def setup_email_routes():
return None
return v[1]
_LIST_STALE = {}
def _list_stale_get(key):
return _LIST_STALE.get(key)
def _list_stale_put(key, value):
_LIST_STALE[key] = value
if len(_LIST_STALE) > 96:
for k in list(_LIST_STALE.keys())[:-48]:
_LIST_STALE.pop(k, None)
_LIST_REFRESHING = set()
def _list_cache_put(key, value):
_LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value)
# Also snapshot the last-good response for stale-while-revalidate.
_list_stale_put(key, value)
# Cap size
if len(_LIST_CACHE) > 64:
for k in list(_LIST_CACHE.keys())[:-32]:
_LIST_CACHE.pop(k, None)
def _schedule_list_refresh(sync_args, cache_key, folder, account_id, owner):
"""Refresh a list snapshot off the event loop, at most once per key.
Runs the blocking IMAP fetch in a daemon thread, then repopulates both
the TTL cache and the stale snapshot so the next list view is warm.
"""
if cache_key in _LIST_REFRESHING:
return
_LIST_REFRESHING.add(cache_key)
def _run():
try:
refreshed = _list_emails_sync(*sync_args)
if refreshed and not refreshed.get("error"):
_list_cache_put(cache_key, refreshed)
try:
_schedule_recent_email_warm(
refreshed.get("emails") or [], folder, account_id, owner)
except Exception:
pass
except Exception as exc: # pragma: no cover - background best effort
logger.debug("email list background refresh failed: %s", exc)
finally:
_LIST_REFRESHING.discard(cache_key)
_threading.Thread(target=_run, daemon=True, name="email-list-refresh").start()
def _folder_cache_get(account_id, owner):
key = (account_id or "", owner or "")
v = _FOLDER_CACHE.get(key)
@@ -2867,6 +2910,46 @@ def setup_email_routes():
owner, account_id or "", folder, filter, limit, offset, elapsed_ms,
)
return cached
# Stale-while-revalidate: never block a plain list view on IMAP while
# we already hold a recent snapshot or a durable index. Serve it now
# with a `stale` marker and refresh in the background so the next view
# is warm. Blocking IMAP only remains for a genuine cold start with no
# snapshot and no indexed rows (e.g. a brand-new account/folder).
if not from_addr and not has_attachments and not (date_from or date_to):
snapshot = _list_stale_get(ck)
if snapshot is None:
try:
indexed_emails, indexed_total, indexed_at = _email_index_list(
owner, account_id, folder, filter, limit, offset, False,
)
except Exception:
indexed_emails, indexed_total, indexed_at = [], 0, None
if indexed_total:
snapshot = {
"emails": indexed_emails,
"total": indexed_total,
"folder": folder,
"offset": offset,
"sync": {
"source": "index",
"indexed": indexed_total,
"updated_at": indexed_at,
},
}
if snapshot is not None:
payload = dict(snapshot)
payload["emails"] = list(payload.get("emails") or [])
_hide_unlinked_calendar_tags(payload["emails"])
sync_meta = dict(payload.get("sync") or {})
sync_meta["source"] = "cache_stale" if _list_stale_get(ck) is not None else "index"
sync_meta["stale"] = True
payload["sync"] = sync_meta
_schedule_list_refresh(
(folder, limit, offset, filter, account_id, from_addr,
bool(has_attachments), owner, True, date_from or "", date_to or ""),
ck, folder, account_id, owner,
)
return payload
result = await _asyncio.to_thread(
_list_emails_sync, folder, limit, offset, filter, account_id, from_addr,
bool(has_attachments), owner, manual_refresh, date_from or "", date_to or "",
@@ -6717,7 +6800,7 @@ def setup_email_routes():
await queue.put({'type': 'result', **result})
task = asyncio.create_task(run())
try:
while True:
while not _shutdown.is_shutting_down():
item = await queue.get()
yield 'data: ' + json.dumps(item) + '\n\n'
if item.get('type') == 'result':
File diff suppressed because it is too large Load Diff
+22 -7
View File
@@ -473,7 +473,7 @@ def _truthy(value: str | None) -> bool:
_ENDPOINT_KINDS = {"auto", "local", "api", "proxy"}
_REFRESH_MODES = {"auto", "manual", "disabled"}
_MODEL_TOOL_MODES = {"none", "compact", "full"}
_MODEL_TOOL_MODES = {"none", "compact", "full", "spec"}
_MODEL_TOOL_MODE_ALIASES = {
"regular": "full",
"odysseus_compact": "compact",
@@ -553,10 +553,10 @@ def _endpoint_refresh_timeout(ep: Any, category: str) -> float:
val = 0
if val > 0:
return float(max(1, min(60, val)))
# llama.cpp and other local OpenAI-compatible servers can block briefly
# while warming/loading. A 2s local timeout makes working endpoints flicker
# offline before /v1/models is ready.
return 10.0 if category == "local" else 2.0
# Keep the background refresh short so a single dead endpoint cannot stall
# the model picker. A timed-out refresh never clears a non-empty model
# cache, so a briefly-warming local server is simply retried next round.
return 2.0
def _manual_refresh_timeout(ep: Any, category: str, requested: Any = None) -> float:
@@ -1511,6 +1511,9 @@ def setup_model_routes(model_discovery):
_refresh_inflight = {"v": False, "done": None} # coarse single-flight guard
_REFRESH_FAILURE_BASE = 300.0
_REFRESH_FAILURE_MAX = 3600.0
# Even a user-triggered refresh must not re-probe an endpoint that just
# failed/timed out on every click; keep a short cooldown.
_REFRESH_FAILURE_MIN = 60.0
def _refresh_key(base: str, api_key: Optional[str], provider_auth_id: Optional[str] = None) -> str:
# Session-backed endpoints carry no static key; include their auth id
@@ -1558,14 +1561,20 @@ def setup_model_routes(model_discovery):
if mode in ("manual", "disabled") and not force:
return False, info
fails = int(state.get("fail_count") or 0)
if fails and not force:
if fails:
last_failure = float(state.get("last_failure") or 0.0)
empty_local = (
not cached
and category == "local"
and str(getattr(ep, "id", "") or "").startswith("local-")
)
if now - last_failure < _failure_delay(fails, empty_local=empty_local):
delay = _failure_delay(fails, empty_local=empty_local)
if force:
# A user-triggered refresh gets a short cooldown instead of the
# long background backoff, but still skips a just-failed
# endpoint so repeated clicks cannot stall the picker.
delay = min(delay, _REFRESH_FAILURE_MIN)
if now - last_failure < delay:
return False, info
if cached and not force:
interval = _endpoint_refresh_interval(ep, category)
@@ -3089,4 +3098,10 @@ def setup_model_routes(model_discovery):
router._should_refresh_endpoint = _should_refresh_endpoint
router._search_endpoint_catalog = search_endpoint_catalog
# Exposed for tests: the in-memory backoff state, its key function, and the
# background refresh driver (probes endpoints in parallel with the short
# per-endpoint timeout).
router._refresh_state = _refresh_state
router._refresh_key = _refresh_key
router._refresh_caches_bg = _refresh_caches_bg
return router
+37
View File
@@ -914,6 +914,43 @@ VARIABLE_NOTES: dict[str, tuple[str, str, str]] = {
"Overrides the source commit reported for runtime provenance, for builds "
"that ship without a git directory.",
),
# -- Photo editor and image operations -----------------------------------
"ODYSSEUS_EDITOR_AGENT_BASES": (
"Model routing and providers", USER,
"Comma-separated base URLs for the image-editor agent backends; a single "
"base URL in the model endpoint config wins over this list.",
),
"ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS": (
"Model routing and providers", USER,
"Set to 1, true, yes or on to allow local image-generation and editing "
"backends to be reachable as image providers.",
),
"ODYSSEUS_IMAGE_OPS_BASE": (
"Model routing and providers", USER,
"Base URL of the image-operations backend (inpaint, outpainting and style "
"transfer); a trailing slash is stripped before use.",
),
"ODYSSEUS_IMAGE_EDIT_MAX_SIDE": (
"Upload and media limits", USER,
"Maximum pixel side (width or height) an edited image is downscaled to "
"before it is sent to the image backend.",
),
"ODYSSEUS_IMAGE_EDIT_MAX_STEPS": (
"Upload and media limits", USER,
"Maximum number of iterative edit steps the image backend may perform in "
"a single request.",
),
"ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT": (
"Upload and media limits", USER,
"Seconds the outpainting proxy waits for the image backend before giving "
"up on a single request.",
),
"ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT": (
"Deployment and first run", INTERNAL,
"Seconds the server waits for in-flight requests and background loops to "
"finish during a graceful shutdown.",
),
}
+61
View File
@@ -0,0 +1,61 @@
"""In-flight activity counters (heavy image ops) for safe restarts/observability.
A deploy restart kills any in-flight long image op (a diffusion edit is 60-120 s),
so the operator needs to know whether the process is busy before issuing
``systemctl --user restart``. These counters are process-local and cheap.
"""
from __future__ import annotations
import threading
import time
from contextlib import contextmanager
from typing import Dict
_lock = threading.Lock()
_image_ops = 0
_image_ops_by_name: Dict[str, int] = {}
_started_at = time.time()
def begin_image_op(name: str = "image") -> None:
global _image_ops
with _lock:
_image_ops += 1
_image_ops_by_name[name] = _image_ops_by_name.get(name, 0) + 1
def end_image_op(name: str = "image") -> None:
global _image_ops
with _lock:
_image_ops = max(0, _image_ops - 1)
if name in _image_ops_by_name:
_image_ops_by_name[name] -= 1
if _image_ops_by_name[name] <= 0:
_image_ops_by_name.pop(name, None)
@contextmanager
def image_op(name: str = "image"):
begin_image_op(name)
try:
yield
finally:
end_image_op(name)
def activity() -> Dict[str, object]:
with _lock:
return {
"image_ops": _image_ops,
"image_ops_by_name": dict(_image_ops_by_name),
"uptime_s": round(time.time() - _started_at, 1),
}
def heavy_image_ops_active() -> int:
with _lock:
return _image_ops
def busy() -> bool:
return heavy_image_ops_active() > 0
+5
View File
@@ -29,6 +29,7 @@ from .interaction_tools import AskUserTool, UpdatePlanTool
from .model_interaction_tools import ChatWithModelTool, AskTeacherTool, ListModelsTool
from .bg_job_tools import ManageBgJobsTool
from .session_tools import CreateSessionTool, ListSessionsTool, SendToSessionTool, ManageSessionTool
from .editor_tools import PlanEditTool, RunEditorCommandTool, GetEditorScreenshotTool, AiEditImageTool
from .admin_tools import (
ADMIN_TOOL_HANDLERS,
do_manage_endpoints, do_manage_mcp, do_manage_webhooks,
@@ -72,6 +73,10 @@ TOOL_HANDLERS = {
"list_sessions": ListSessionsTool().execute,
"send_to_session": SendToSessionTool().execute,
"manage_session": ManageSessionTool().execute,
"plan_edit": PlanEditTool().execute,
"run_editor_command": RunEditorCommandTool().execute,
"get_editor_screenshot": GetEditorScreenshotTool().execute,
"ai_edit_image": AiEditImageTool().execute,
}
# Config/integration admin tools (manage_endpoints/mcp/webhooks/tokens/settings).
TOOL_HANDLERS.update(ADMIN_TOOL_HANDLERS)
+646
View File
@@ -0,0 +1,646 @@
"""In-process agent tools for the scoped ``photo_editor`` capability family.
These handlers run as the logged-in owner and drive the image editor over the
owner-scoped in-process bridge (``/api/editor/bridge/*``) plus the app's
existing image transforms (``/api/image/*``). They are only offered to the
model when the ``photo_editor`` family is active (the Quick Edit popup or an
explicit photo-edit chat turn), never in the general tool list.
Loopback calls carry the internal token and impersonate ``owner`` (middleware
app.py), so the editor bridge resolves the same ``owner:session`` scope the
browser subscriber uses.
"""
from __future__ import annotations
import base64
import hashlib
import io
import json
import logging
import re
import time
import uuid
from pathlib import Path
from typing import Any, Dict, List, Optional
import httpx
from src.tools._common import _INTERNAL_BASE, _internal_headers
logger = logging.getLogger(__name__)
def _provider_message() -> str:
try:
from src.ai_interaction import IMAGE_PROVIDER_REQUIRED_MESSAGE
return IMAGE_PROVIDER_REQUIRED_MESSAGE
except Exception:
return (
"Removing/adding objects needs an image generation provider; enable one "
"in Settings > Models."
)
def _provider_available(owner: Optional[str]) -> bool:
try:
from src.ai_interaction import image_editing_provider_available
return bool(image_editing_provider_available(owner))
except Exception:
return True
# Local diffusion/image servers can take 90-110s per image (8 steps) and
# longer on a cold start; the previous 120s default timed out mid-generation.
_EDITOR_TIMEOUT = 300.0
# A mask covering less than this fraction of the image cannot drive a useful
# regional inpaint (tiny figures, empty SAM output): fall back to grounding or
# whole-image instruction mode instead.
_MIN_MASK_FRACTION = 0.002
_AI_OPS = {
"remove-bg": "/api/image/remove-bg",
"upscale": "/api/image/upscale-local",
"denoise": "/api/image/denoise",
"enhance-face": "/api/image/enhance-face",
"inpaint": "/api/image/inpaint",
"generative": "/api/image/inpaint",
}
_PLACE_OP = "layers.placeImage"
def _parse(content: Any) -> Dict[str, Any]:
raw = content if isinstance(content, str) else json.dumps(content or {})
try:
parsed = json.loads(raw) if raw.strip() else {}
except (ValueError, TypeError):
parsed = {}
return parsed if isinstance(parsed, dict) else {}
def _session(args: Dict[str, Any], ctx: Dict[str, Any]) -> str:
return str(args.get("session") or args.get("editor_session") or ctx.get("session_id") or "default")
async def _post(path: str, payload: Dict[str, Any], owner: Optional[str], timeout: float = _EDITOR_TIMEOUT) -> Dict[str, Any]:
headers = {**_internal_headers(owner), "Content-Type": "application/json"}
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.post(f"{_INTERNAL_BASE}{path}", json=payload, headers=headers)
resp.raise_for_status()
try:
return resp.json()
except ValueError:
return {"raw": resp.text}
def _as_data_url(value: Any) -> Optional[str]:
if not isinstance(value, str) or not value:
return None
if value.startswith("data:"):
return value
if value.startswith("/"):
return None
return f"data:image/png;base64,{value}"
def _data_url_bytes(data_url: str) -> bytes:
_, _, b64 = data_url.partition(",")
return base64.b64decode(b64)
def _persist_png(image_bytes: bytes, owner: Optional[str], session_id: Optional[str], prompt: str, model: str) -> Dict[str, Any]:
"""Save an edited/generated PNG and return {image_id, image_url, image_size}."""
from core.database import GalleryImage, SessionLocal
from src.constants import GENERATED_IMAGES_DIR
db = SessionLocal()
try:
root = Path(GENERATED_IMAGES_DIR).resolve()
root.mkdir(parents=True, exist_ok=True)
filename = f"{uuid.uuid4().hex[:12]}.png"
(root / filename).write_bytes(image_bytes)
new_id = str(uuid.uuid4())
width = height = 0
try:
from PIL import Image
with Image.open(io.BytesIO(image_bytes)) as img:
width, height = img.size
except Exception:
pass
size = f"{width}x{height}" if width and height else ""
# The editor bridge uses an owner-scoped session id (e.g. "pewds:3")
# that is not a chat-sessions row. GalleryImage.session_id is a FK to
# sessions.id, so drop an unknown id instead of raising IntegrityError.
if session_id:
from core.database import Session as _ChatSession
try:
known = db.query(_ChatSession.id).filter(_ChatSession.id == session_id).first()
except Exception:
known = None
if known is None:
session_id = None
derived = GalleryImage(
id=new_id,
filename=filename,
prompt=prompt or None,
caption=None,
model=model,
size=size or None,
quality=None,
tags=None,
ai_tags=None,
session_id=session_id,
album_id=None,
owner=owner,
file_hash=hashlib.sha256(image_bytes).hexdigest(),
file_size=len(image_bytes),
width=width or None,
height=height or None,
)
db.add(derived)
db.commit()
return {"image_id": new_id, "image_url": f"/api/generated-image/{filename}", "image_size": size or None}
except Exception:
db.rollback()
raise
finally:
db.close()
async def _place_layer(data_url: str, name: str, owner: Optional[str], session: str, *, masked: bool = False, blend: str = "normal", opacity: float = 100.0, mode: Optional[str] = None) -> bool:
params: Dict[str, Any] = {
"dataUrl": data_url,
"name": name,
"masked": bool(masked),
"blend": blend or "normal",
"opacity": opacity,
}
# Forward the edit mode so the client can build a difference mask for
# instruction-mode results: the whole image was regenerated under an
# instruction prompt, so untouched pixels must stay pixel-identical (the
# client keeps only the changed region) instead of the new layer overwriting
# the entire canvas. Omit the keys entirely when no mode is given so the
# client keeps its current behavior (e.g. generate_image must not diff-mask).
if mode:
params["mode"] = mode
params["differenceMask"] = mode == "instruction"
if mode == "instruction":
# Exclude a thin edge band from the client difference mask: full-frame
# regenerations can shift pixels along the border.
params["edgeExcludePx"] = 3
payload = {
"session": session,
"op": _PLACE_OP,
"params": params,
"wait": True,
"timeout": 60,
}
try:
data = await _post("/api/editor/bridge/command", payload, owner, timeout=90)
return data.get("ok") is not False
except Exception as exc: # pragma: no cover - defensive
logger.warning("editor layer placement failed: %s", exc)
return False
async def _fetch_selection_mask(owner: Optional[str], session: str) -> Optional[str]:
"""Return the editor's current selection as a data-URL mask, or None.
Generative ops (inpaint/generative) need a mask; the planner/judge may not
have supplied one, so fall back to whatever is selected in the editor.
"""
payload = {
"session": session,
"op": "selection.exportMask",
"params": {},
"wait": True,
"timeout": 30,
}
try:
data = await _post("/api/editor/bridge/command", payload, owner, timeout=45)
except Exception as exc: # pragma: no cover - defensive
logger.warning("editor selection mask fetch failed: %s", exc)
return None
result = data.get("result") if isinstance(data, dict) else None
if isinstance(result, dict):
return _as_data_url(result.get("mask"))
return None
def _mask_white_fraction(data_url: Optional[str]) -> float:
"""Fraction of a grayscale mask that is selected (white, >=128)."""
if not data_url:
return 0.0
try:
from PIL import Image as _PILImage
with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))).convert("L") as im:
total = im.width * im.height
if not total:
return 0.0
hist = im.histogram()
return sum(hist[128:]) / total
except Exception:
return 0.0
def _image_is_blank(data_url: str) -> bool:
"""True when a result has an alpha channel that is essentially transparent."""
try:
from PIL import Image as _PILImage
with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))) as im:
if im.mode not in ("RGBA", "LA") and "transparency" not in im.info:
return False
alpha = im.convert("RGBA").getchannel("A")
hist = alpha.histogram()
total = sum(hist)
return total > 0 and (sum(hist[1:]) / total) < 0.001
except Exception:
return False
def _rect_mask_data_url(image_data_url: str, box: List[float], *, pad_frac: float = 0.15) -> Optional[str]:
"""Build a rectangular mask data URL from a [x1,y1,x2,y2] box, dilated.
Used when the grounding detector finds an object but SAM returns an empty
mask (tiny figures): a dilated box guarantees a usable region.
"""
try:
from PIL import Image as _PILImage, ImageDraw
with _PILImage.open(io.BytesIO(_data_url_bytes(image_data_url))) as im:
w, h = im.size
x1, y1, x2, y2 = (float(v) for v in box)
bw, bh = x2 - x1, y2 - y1
x1, y1 = max(0.0, x1 - bw * pad_frac), max(0.0, y1 - bh * pad_frac)
x2, y2 = min(float(w), x2 + bw * pad_frac), min(float(h), y2 + bh * pad_frac)
mask = _PILImage.new("L", (w, h), 0)
ImageDraw.Draw(mask).rectangle([x1, y1, x2, y2], fill=255)
buf = io.BytesIO()
mask.save(buf, "PNG")
return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
except Exception:
return None
async def _ground_box(owner: Optional[str], image_data_url: str, label: str) -> Optional[List[float]]:
"""Use the in-app grounding detector (OWL-ViT via /api/image/mask) for a box."""
try:
data = await _post("/api/image/mask", {"image": image_data_url, "text": label}, owner, timeout=90)
except Exception as exc: # pragma: no cover - defensive
logger.warning("editor grounding box failed: %s", exc)
return None
box = data.get("bbox") if isinstance(data, dict) else None
if isinstance(box, (list, tuple)) and len(box) == 4:
try:
return [float(v) for v in box]
except Exception:
return None
return None
def _ensure_keep_clause(text: str) -> str:
"""Append a keep-the-subject clause when the prompt lacks one."""
body = str(text or "").strip()
if not body:
return body
if re.search(r"\b(keep|preserve|leave|unchanged|identical|same)\b", body, re.I):
return body
return body.rstrip(". ") + ". Keep the main subject and everything else in the image exactly the same."
def _mask_label(args: Dict[str, Any], prompt: str) -> str:
"""Best-effort object noun for grounding a regional removal."""
label = str(args.get("label") or args.get("object") or "").strip()
if label:
return label
m = re.search(
r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out)\s+(?:the\s+|a\s+|an\s+)?"
r"([a-z][a-z \-]{1,30}?)(?:\s+(?:on|in|at|from|to|near|behind|beside|by)\b|[.,;!?]|$)",
prompt or "",
re.I,
)
if m:
noun = m.group(1).strip().lower()
if noun and noun not in {"background", "bg"}:
return noun
for noun in ("person", "people", "man", "woman", "car", "dog", "sign", "logo"):
if re.search(rf"\b{re.escape(noun)}\b", prompt or "", re.I):
return "person" if noun in {"person", "people", "man", "woman"} else noun
return "object"
class PlanEditTool:
"""Plan an image edit as ordered editor operations (no side effects)."""
async def execute(self, content: Any, ctx: Dict[str, Any]):
args = _parse(content)
prompt = str(args.get("prompt") or "").strip()
if not prompt:
return "plan_edit: invalid", {"error": "plan_edit needs a non-empty `prompt`.", "exit_code": 1}
state = args.get("state") if isinstance(args.get("state"), dict) else {}
try:
data = await _post(
"/api/editor/plan",
{"prompt": prompt, "state": state, "mode": "plan", "session": _session(args, ctx)},
ctx.get("owner"),
)
except Exception as exc:
return "plan_edit: error", {"error": f"plan_edit failed: {exc}", "exit_code": 1}
steps = data.get("steps") or []
reply = str(data.get("reply") or "")
return f"plan_edit: {len(steps)} step(s)", {
"output": reply or f"Planned {len(steps)} editor step(s).",
"reply": reply,
"planner": data.get("planner"),
"steps": steps,
"exit_code": 0,
}
class RunEditorCommandTool:
"""Run a single validated editor op in the browser via the owner bridge."""
async def execute(self, content: Any, ctx: Dict[str, Any]):
args = _parse(content)
op = str(args.get("op") or "").strip()
if not op:
return "run_editor_command: invalid", {"error": "run_editor_command needs `op`.", "exit_code": 1}
params = args.get("params") if isinstance(args.get("params"), dict) else {}
payload = {
"session": _session(args, ctx),
"op": op,
"params": params,
"wait": bool(args.get("wait", True)),
"timeout": float(args.get("timeout") or 60),
}
try:
data = await _post("/api/editor/bridge/command", payload, ctx.get("owner"))
except Exception as exc:
return f"run_editor_command: {op}", {"error": f"run_editor_command failed: {exc}", "exit_code": 1}
ok = data.get("ok")
result = data.get("result")
summary = json.dumps({"ok": ok, "error": data.get("error"), "result": result}, ensure_ascii=False)
return f"run_editor_command: {op}", {
"output": summary[:4000],
"result": result,
"screenshot": data.get("screenshot"),
"ok": ok,
"exit_code": 0 if ok is not False else 1,
}
class GetEditorScreenshotTool:
"""Capture the current editor canvas as a data URL."""
async def execute(self, content: Any, ctx: Dict[str, Any]):
args = _parse(content)
payload = {
"session": _session(args, ctx),
"op": "screenshot",
"params": {},
"wait": True,
"timeout": float(args.get("timeout") or 30),
}
try:
data = await _post("/api/editor/bridge/command", payload, ctx.get("owner"), timeout=45)
except Exception as exc:
return "get_editor_screenshot: error", {"error": f"get_editor_screenshot failed: {exc}", "exit_code": 1}
result = data.get("result") if isinstance(data.get("result"), dict) else {}
shot = data.get("screenshot") or result.get("dataUrl")
if not shot:
return "get_editor_screenshot: unavailable", {
"error": "No editor screenshot was returned (is the editor open?).",
"exit_code": 1,
}
return "get_editor_screenshot", {"output": "Captured the editor canvas.", "screenshot": shot, "exit_code": 0}
class AiEditImageTool:
"""Apply an AI image transform (remove-bg/upscale/denoise/enhance/inpaint)."""
async def execute(self, content: Any, ctx: Dict[str, Any]):
args = _parse(content)
_t = {"start": time.perf_counter()}
op = str(args.get("op") or "").strip().lower()
if op not in _AI_OPS:
return "ai_edit_image: invalid", {
"error": f"ai_edit_image `op` must be one of {sorted(_AI_OPS)}.",
"exit_code": 1,
}
owner = ctx.get("owner")
if op in ("inpaint", "generative") and not _provider_available(owner):
return "ai_edit_image: provider required", {
"error": _provider_message(),
"failure_kind": "image_provider_unavailable",
"needs_provider": True,
"exit_code": 1,
}
session = _session(args, ctx)
image = _as_data_url(args.get("image"))
if image is None and isinstance(args.get("image"), str) and args["image"].startswith("/"):
# A same-origin gallery/attachment URL: fetch it (owner-scoped) as bytes.
try:
async with httpx.AsyncClient(timeout=60) as client:
resp = await client.get(f"{_INTERNAL_BASE}{args['image']}", headers=_internal_headers(owner))
resp.raise_for_status()
image = _as_data_url("data:image/png;base64," + base64.b64encode(resp.content).decode())
except Exception:
image = None
used_canvas = False
if not image:
shot = await GetEditorScreenshotTool().execute(json.dumps({"session": session}), ctx)
image = (shot[1] or {}).get("screenshot") if isinstance(shot, tuple) else None
used_canvas = bool(image)
if not image:
return "ai_edit_image: no image", {
"error": "ai_edit_image needs an `image` (data URL or same-origin URL) or an open editor.",
"exit_code": 1,
}
_t["image"] = time.perf_counter()
payload: Dict[str, Any] = {"image": image}
model_choice = str(args.get("model") or "").strip()
mask_used = False
place_mode: Optional[str] = None
if op == "upscale":
payload["scale"] = int(args.get("scale") or 2)
elif op == "denoise":
payload["strength"] = float(args.get("strength") or 0.5)
elif op in ("inpaint", "generative"):
payload["prompt"] = str(args.get("prompt") or "").strip()
if model_choice:
# inpaint_proxy reads `_model` to route to a specific model.
payload["_model"] = model_choice
mode = str(args.get("mode") or "instruction").strip().lower()
_steps = int(args.get("steps") or 0)
if _steps > 0:
payload["steps"] = _steps
mask_used = False
mask = None
if mode == "masked":
mask = _as_data_url(args.get("mask"))
if not mask:
mask = await _fetch_selection_mask(owner, session)
# An empty/tiny SAM mask on small figures is useless: drop it.
if mask and _mask_white_fraction(mask) < _MIN_MASK_FRACTION:
mask = None
if not mask:
# Fall back to the grounding detector (OWL-ViT): a dilated
# box mask around the matching object.
box = await _ground_box(owner, image, _mask_label(args, payload["prompt"]))
if box:
candidate = _rect_mask_data_url(image, box, pad_frac=0.15)
if candidate and _mask_white_fraction(candidate) >= _MIN_MASK_FRACTION:
mask = candidate
if mask:
payload["mask"] = mask
payload["mode"] = "masked"
mask_used = True
else:
# Nothing usable to mask -> whole-image instruction fallback
# rather than placing a transparent/empty layer.
mode = "instruction"
if mode != "masked":
# Instruction mode: whole-image edit with no tight mask. Send a
# full-white mask so the edit endpoint accepts the request and
# regenerates the whole frame under the instruction prompt, which
# names what must stay unchanged. This avoids the mask-compositing
# halos that loose selections caused.
payload["mode"] = "instruction"
payload.pop("mask", None)
payload["prompt"] = _ensure_keep_clause(payload["prompt"])
try:
from PIL import Image as _PILImage
_raw = _data_url_bytes(image)
_w, _h = _PILImage.open(io.BytesIO(_raw)).size
_mbuf = io.BytesIO()
_PILImage.new("L", (_w, _h), 255).save(_mbuf, "PNG")
payload["mask"] = "data:image/png;base64," + base64.b64encode(_mbuf.getvalue()).decode()
except Exception:
pass
place_mode = "masked" if mask_used else "instruction"
_t["prep"] = time.perf_counter()
try:
data = await _post(_AI_OPS[op], payload, owner)
except Exception as exc:
logger.info(
"image-op ai_edit_image op=%s stage=provider FAILED image_ms=%.1f prep_ms=%.1f total_ms=%.1f err=%s",
op, (_t["image"] - _t["start"]) * 1000.0, (_t["prep"] - _t["image"]) * 1000.0,
(time.perf_counter() - _t["start"]) * 1000.0, exc,
)
return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) failed: {exc}", "exit_code": 1}
_t["provider"] = time.perf_counter()
result = data.get("image") or data.get("data")
data_url = _as_data_url(result)
if not data_url:
return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) returned no image.", "exit_code": 1}
if _image_is_blank(data_url):
return f"ai_edit_image: {op}", {
"error": f"ai_edit_image ({op}) produced an empty layer.",
"failure_kind": "empty_result",
"exit_code": 1,
}
info: Dict[str, Any] = {
"output": f"Applied AI edit '{op}'" + (" to the current canvas." if used_canvas else "."),
"op": op,
"model": data.get("model") or model_choice or "",
"exit_code": 0,
}
try:
info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), f"ai_edit_image:{op}", f"ai_edit_image:{op}"))
except Exception as exc:
logger.warning("ai_edit_image persist failed: %s", exc)
_t["persist"] = time.perf_counter()
if bool(args.get("place", True)):
placed = await _place_layer(
data_url,
f"AI {op}",
owner,
session,
masked=mask_used,
blend=str(args.get("blend") or "normal"),
opacity=float(args.get("opacity") or 100),
mode=place_mode,
)
info["placed"] = placed
if placed:
info["output"] += " Placed the result as a new layer."
_t["place"] = time.perf_counter()
logger.info(
"image-op ai_edit_image op=%s mode=%s canvas=%s image_ms=%.1f mask_ms=%.1f "
"model_ms=%.1f persist_ms=%.1f place_ms=%.1f total_ms=%.1f",
op, place_mode or "instruction", used_canvas,
(_t["image"] - _t["start"]) * 1000.0,
(_t["prep"] - _t["image"]) * 1000.0,
(_t["provider"] - _t["prep"]) * 1000.0,
(_t["persist"] - _t["provider"]) * 1000.0,
(_t["place"] - _t["persist"]) * 1000.0,
(_t["place"] - _t["start"]) * 1000.0,
)
return f"ai_edit_image: {op}", info
class GenerateImageLayerTool:
"""Generate an image with the configured image model and place it as a layer."""
async def execute(self, content: str, ctx: dict):
args = _parse(content)
prompt = str(args.get("prompt") or "").strip()
if not prompt:
return "generate_image: invalid", {"error": "generate_image needs a non-empty `prompt`.", "exit_code": 1}
owner = ctx.get("owner")
if not _provider_available(owner):
return "generate_image: provider required", {
"error": _provider_message(),
"failure_kind": "image_provider_unavailable",
"needs_provider": True,
"exit_code": 1,
}
session = _session(args, ctx)
payload: Dict[str, Any] = {"prompt": prompt}
for key in ("model", "size", "quality"):
if args.get(key):
payload[key] = args[key]
try:
from src.ai_interaction import do_generate_image
data = await do_generate_image(
json.dumps(payload, ensure_ascii=False),
session_id=session,
owner=owner,
incognito=True,
)
except Exception as exc:
return "generate_image: error", {"error": f"generate_image failed: {exc}", "exit_code": 1}
result = data.get("image_url") or data.get("image") or data.get("data")
data_url = _as_data_url(result)
if not data_url:
return "generate_image", {"error": "generate_image returned no image.", "exit_code": 1}
info: Dict[str, Any] = {
"output": f"Generated an image for: {prompt[:120]}",
"image_size": data.get("image_size"),
"model": data.get("image_model") or str(args.get("model") or ""),
"exit_code": 0,
}
try:
info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), prompt, "generate_image"))
except Exception as exc:
logger.warning("generate_image persist failed: %s", exc)
if bool(args.get("place", True)):
placed = await _place_layer(
data_url,
str(args.get("name") or "Generated image"),
owner,
session,
masked=bool(args.get("mask")),
blend=str(args.get("blend") or "normal"),
opacity=float(args.get("opacity") or 100),
)
info["placed"] = placed
if placed:
info["output"] += " Placed it as a new layer."
return "generate_image", info
+72 -20
View File
@@ -965,7 +965,7 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O
elif action == "open_panel":
# Open a top-level panel/modal: documents/library, gallery,
# calendar, email, sessions, notes, memories, skills, settings, theme, cookbook.
# calendar, email, sessions, notes, tasks, memories, skills, settings, theme, cookbook.
panel = parts[1].lower() if len(parts) > 1 else ""
view = ""
view_label = ""
@@ -993,6 +993,8 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O
"note": "notes",
"todo": "notes",
"todos": "notes",
"tasks": "tasks",
"task": "tasks",
"memories": "memories",
"memory": "memories",
"brain": "memories",
@@ -1010,7 +1012,7 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O
}
target = _panel_aliases.get(panel)
if not target:
return {"error": f"Unknown panel '{panel}'. Valid: documents, gallery, calendar, email, sessions, notes, memories, skills, settings, theme, cookbook."}
return {"error": f"Unknown panel '{panel}'. Valid: documents, gallery, calendar, email, sessions, notes, tasks, memories, skills, settings, theme, cookbook."}
if target == "cookbook":
cookbook_views = {
"models": ("Search", "models"), "model": ("Search", "models"),
@@ -1155,7 +1157,50 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O
# Image generation
# ---------------------------------------------------------------------------
async def do_generate_image(content: str, session_id: Optional[str] = None, owner: Optional[str] = None) -> Dict:
# Message used whenever a generative image edit (object add/remove/replace,
# image generation) is requested but no image provider is configured. The
# photo-editor agent and the image tools surface this verbatim so the user gets
# a plain, actionable explanation instead of a silent substitute edit.
IMAGE_PROVIDER_REQUIRED_MESSAGE = (
"Removing/adding objects needs an image generation provider; enable one in "
"Settings > Models."
)
def image_editing_provider_available(owner: Optional[str] = None) -> bool:
"""True when an enabled, owner-visible image endpoint can serve AI edits.
Generative operations (object add/remove/replace, image generation) require
a configured image model. Local transforms (remove-bg, upscale, denoise,
enhance-face) do not, but the photo-editor agent gates the whole generative
request on this so it never falls back to deterministic approximations.
"""
try:
from src.settings import load_settings
if not load_settings().get("image_gen_enabled", True):
return False
except Exception:
pass
try:
from src.database import SessionLocal, ModelEndpoint
from src.auth_helpers import owner_filter
db = SessionLocal()
try:
q = db.query(ModelEndpoint).filter(
ModelEndpoint.is_enabled == True, # noqa: E712
ModelEndpoint.model_type == "image",
)
q = owner_filter(q, ModelEndpoint, owner)
return q.first() is not None
finally:
db.close()
except Exception:
return False
async def do_generate_image(content: str, session_id: Optional[str] = None, owner: Optional[str] = None, *, incognito: bool = False) -> Dict:
"""Generate an image using an image-capable model (e.g. gpt-image-1).
Content format:
@@ -1255,6 +1300,14 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
async def _resolve_image_model(model_name: str):
def _call():
# Try a plain by-name lookup first. Gateway endpoints (OpenRouter)
# register as `llm` yet serve image models, and the image-typed
# query's last-resort fallback would otherwise hijack a cloud model
# onto the local image endpoint (gpt-5-image -> qwen-image-edit).
try:
return _resolve_model(model_name, owner=owner)
except ValueError:
pass
try:
return _resolve_model(model_name, owner=owner, model_type="image")
except TypeError as exc:
@@ -1314,7 +1367,7 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
else:
payload["quality"] = "medium"
logger.info(f"Image generation: model={model_id}, size={size}, quality={quality}, prompt={prompt[:80]}")
logger.info(f"Image generation: model={model_id}, size={size}, quality={quality}, prompt={'[private]' if incognito else prompt[:80]}")
try:
# GPT image models can take 30-120s+ depending on quality
@@ -1365,18 +1418,22 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
logger.warning(f"Failed to save gallery record: {_ge}")
return ""
# GPT image models always return b64_json; DALL-E may return url
if img.get("b64_json"):
def _store_image(image_bytes):
if incognito:
return 'data:image/png;base64,' + base64.b64encode(image_bytes).decode('ascii'), None
img_dir = Path(GENERATED_IMAGES_DIR)
img_dir.mkdir(parents=True, exist_ok=True)
filename = f"{uuid.uuid4().hex[:12]}.png"
img_path = img_dir / filename
img_path.write_bytes(base64.b64decode(img.get("b64_json")))
img_path.write_bytes(image_bytes)
if owner:
from src.generated_images import register_generated_image
register_generated_image(filename, owner)
image_url = f"/api/generated-image/{filename}"
image_id = _save_to_gallery(filename)
return f"/api/generated-image/{filename}", _save_to_gallery(filename)
# Private results stay in the response, never in the gallery or on disk.
if img.get("b64_json"):
image_url, image_id = _store_image(base64.b64decode(img['b64_json']))
elif img.get("url"):
# Download external URL and save locally (DALL-E returns temp URLs)
@@ -1390,16 +1447,7 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
try:
dl_resp = httpx.get(result_url, timeout=60)
if dl_resp.status_code == 200:
img_dir = Path(GENERATED_IMAGES_DIR)
img_dir.mkdir(parents=True, exist_ok=True)
filename = f"{uuid.uuid4().hex[:12]}.png"
img_path = img_dir / filename
img_path.write_bytes(dl_resp.content)
if owner:
from src.generated_images import register_generated_image
register_generated_image(filename, owner)
image_url = f"/api/generated-image/{filename}"
image_id = _save_to_gallery(filename)
image_url, image_id = _store_image(dl_resp.content)
else:
image_url = result_url # fallback to external URL
except Exception as _dl_e:
@@ -1436,6 +1484,7 @@ async def do_edit_image(
size: str = "auto",
quality: str = "medium",
progress_callback: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
incognito: bool = False,
) -> Dict:
"""Edit an uploaded image using the configured image endpoint."""
import base64
@@ -1509,7 +1558,7 @@ async def do_edit_image(
request_id = uuid.uuid4().hex
payload["request_id"] = request_id
logger.info("Image edit: model=%s, size=%s, quality=%s, image=%s, prompt=%s", model_id, size, quality, path.name, prompt[:80])
logger.info("Image edit: model=%s, size=%s, quality=%s, image=%s, prompt=%s", model_id, size, quality, '[private]' if incognito else path.name, '[private]' if incognito else prompt[:80])
def _save_edited_image_to_gallery(filename: str) -> str:
try:
@@ -1539,6 +1588,9 @@ async def do_edit_image(
from PIL import Image
with Image.open(BytesIO(image_bytes)) as output:
size = f"{output.width}x{output.height}"
image_mime = Image.MIME.get(output.format, 'image/png')
if incognito:
return 'data:' + image_mime + ';base64,' + base64.b64encode(image_bytes).decode('ascii'), ''
img_dir = Path(GENERATED_IMAGES_DIR)
img_dir.mkdir(parents=True, exist_ok=True)
filename = f"{uuid.uuid4().hex[:12]}{suffix}"
+498
View File
@@ -0,0 +1,498 @@
"""Editor agent/planner model resolution.
The image-editor Quick Edit planner (`/api/editor/plan`, `/api/editor/agent`)
runs on an explicitly configured local vision model when one is reachable,
with the configured chat model as a fallback. This module owns:
* the local model base URLs / model id (settings-overridable),
* an ordered candidate list (per-request ``agent_model`` override, then the
local Qwen3.5-9B, then the hosted default/utility chat model),
* a vision fallback chain: local model WITH the canvas screenshot, else a
text-only call whose image is described by a separate VL caption model,
* a lightweight reachability probe for the configured local bases.
Everything is best-effort: when no local server is reachable the planner
transparently falls back to the previously-used hosted chat model, so the
editor agent never regresses to "no plan".
"""
from __future__ import annotations
import contextvars
import json
import logging
import os
import re
import threading
from typing import Any, Dict, List, Optional, Tuple
import httpx
logger = logging.getLogger(__name__)
# Local endpoints are opt-in through settings or ODYSSEUS_EDITOR_AGENT_BASES.
DEFAULT_LOCAL_BASES = []
DEFAULT_LOCAL_MODEL = "Qwen/Qwen3.5-9B"
# Vision caption model used when the editor-agent model cannot accept images.
DEFAULT_CAPTION_BASE = ""
DEFAULT_CAPTION_MODEL = "Qwen/Qwen3.5-9B"
# Served-model-id cache per base (populated on first use). The local servers 400
# with "Failed to load model" when asked for an id they do not serve (e.g. the
# client's default "Qwen3.5-9B-MLX-8bit", or a stale caption model), so we
# resolve the requested id against GET {base}/models and fall back to one that
# is actually served.
_MODELS_CACHE: Dict[str, set] = {}
_CAPTION_INSTRUCTION = (
"Describe this image in one or two sentences for a photo editor: the main "
"subject, the background/scene, lighting, dominant colours, and any obvious "
"flaws (blown highlights, colour cast, clutter, blur)."
)
def _setting(key: str, default: Any = "") -> Any:
try:
from src.settings import load_settings
value = load_settings().get(key)
return value if value not in (None, "") else default
except Exception:
return default
def local_bases() -> List[str]:
override = _setting("editor_agent_base_url")
if override:
return [str(override)]
env = os.getenv("ODYSSEUS_EDITOR_AGENT_BASES")
if env:
bases = [b.strip() for b in env.split(",") if b.strip()]
if bases:
return bases
return list(DEFAULT_LOCAL_BASES)
def local_model() -> str:
return str(_setting("editor_agent_model", DEFAULT_LOCAL_MODEL))
def caption_base() -> str:
return str(_setting("editor_caption_base_url", DEFAULT_CAPTION_BASE))
def caption_model() -> str:
return str(_setting("editor_caption_model", DEFAULT_CAPTION_MODEL))
# Per-request / per-plan context (contextvars keep this correct under the
# async event loop without sharing state between concurrent editor turns).
_override: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
"editor_agent_model_override", default=None
)
_last_model: contextvars.ContextVar[Optional[Tuple[str, str]]] = contextvars.ContextVar(
"editor_agent_last_model", default=None
)
def set_override(value: Optional[str]) -> None:
_override.set(value or None)
def get_override() -> Optional[str]:
return _override.get()
def set_last_model(model: str, source: str) -> None:
_last_model.set((model, source))
def get_last_model() -> Optional[Tuple[str, str]]:
return _last_model.get()
def _chat_url(base: str) -> str:
b = str(base or "").rstrip("/")
if b.endswith("/chat/completions"):
return b
return b + "/chat/completions"
def _split_override(override: str) -> Tuple[str, str]:
"""Parse ``base::model`` (or a bare model id)."""
raw = str(override or "").strip()
if "::" in raw:
base, model = raw.split("::", 1)
return base.strip(), model.strip()
return "", raw
def _has_image(messages: List[dict]) -> bool:
for msg in messages:
content = msg.get("content")
if isinstance(content, list):
for part in content:
if isinstance(part, dict) and part.get("type") == "image_url":
return True
return False
def _first_image(messages: List[dict]) -> Optional[str]:
for msg in messages:
content = msg.get("content")
if isinstance(content, list):
for part in content:
if isinstance(part, dict) and part.get("type") == "image_url":
url = (part.get("image_url") or {}).get("url")
if isinstance(url, str):
return url
return None
def _strip_images(messages: List[dict]) -> List[dict]:
out: List[dict] = []
for msg in messages:
content = msg.get("content")
if isinstance(content, list):
text = "\n".join(
str(part.get("text") or "")
for part in content
if isinstance(part, dict) and part.get("type") == "text"
)
out.append({"role": msg.get("role", "user"), "content": text})
else:
out.append({"role": msg.get("role", "user"), "content": content})
return out
def _with_caption(messages: List[dict], caption: str) -> List[dict]:
note = f"\n\n[Canvas screenshot caption: {caption}]"
out = [dict(m) for m in messages]
for msg in reversed(out):
if msg.get("role") == "user":
msg["content"] = str(msg.get("content") or "") + note
break
return out
async def _post_chat(
base: str,
model: str,
messages: List[dict],
headers: Optional[dict],
timeout: float,
max_tokens: int = 1024,
response_format: Optional[dict] = None,
) -> Optional[str]:
if not base or not model:
return None
payload: Dict[str, Any] = {
"model": model,
"messages": messages,
"temperature": 0.2,
"max_tokens": max_tokens,
"stream": False,
}
# Constrain the planner to a strict JSON ops schema (structured outputs /
# grammar) so the model can only emit valid editor ops. Unknown servers
# ignore the field; the server-side schema validation is the real enforcer.
if isinstance(response_format, dict):
payload["response_format"] = response_format
try:
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.post(_chat_url(base), json=payload, headers=headers or {})
if resp.status_code != 200:
logger.debug(
"editor agent chat %s -> %s (%s)", base, resp.status_code, resp.text[:200]
)
return None
data = resp.json()
except Exception as exc:
logger.debug("editor agent chat %s failed: %s", base, exc)
return None
try:
return data["choices"][0]["message"]["content"]
except Exception:
return None
async def _served_models(base: str, timeout: float = 3.0) -> set:
"""Return the set of model ids a base serves (cached; empty on failure)."""
if not base:
return set()
cached = _MODELS_CACHE.get(base)
if cached is not None:
return cached
ids: set = set()
try:
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.get(base.rstrip("/") + "/models")
if resp.status_code == 200:
ids = {
str(m.get("id"))
for m in (resp.json().get("data") or [])
if isinstance(m, dict) and m.get("id")
}
except Exception:
ids = set()
_MODELS_CACHE[base] = ids
return ids
async def _resolve_model(base: str, requested: str, fallback: str) -> str:
"""Return a model id ``base`` actually serves.
If the requested id is served, keep it; otherwise use ``fallback`` (when
served) or any served id. When the served list is unknown (server down),
keep the requested id (or ``fallback``).
"""
requested = str(requested or "").strip()
fallback = str(fallback or "").strip()
ids = await _served_models(base)
if ids:
if requested and requested in ids:
return requested
if fallback in ids:
return fallback
return sorted(ids)[0]
return requested or fallback
async def caption_image(data_url: str, owner: Optional[str] = None, timeout: float = 90.0) -> Optional[str]:
if not isinstance(data_url, str) or not data_url.startswith("data:image/"):
return None
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": _CAPTION_INSTRUCTION},
{"type": "image_url", "image_url": {"url": data_url}},
],
}
]
base = caption_base()
if not base:
return None
model = await _resolve_model(base, caption_model(), local_model())
return await _post_chat(base, model, messages, None, timeout, max_tokens=200)
_DONE_RE = re.compile(r"^\s*DONE\b", re.I)
_RETRY_RE = re.compile(r"RETRY:?\s*(.+)", re.I | re.S)
def _parse_self_check(text: str) -> Dict[str, Any]:
s = str(text or "").strip()
if _DONE_RE.match(s):
return {"done": True, "issues": [], "retry_hint": "", "text": s}
m = _RETRY_RE.search(s)
if m or re.search(r"\bretry\b", s, re.I):
hint = (m.group(1).strip() if m else s) or s
return {"done": False, "issues": [hint], "retry_hint": hint, "text": s}
return {"done": True, "issues": [], "retry_hint": "", "text": s}
async def self_check(
before: Any,
after: Any,
prompt: str,
model: Optional[str] = None,
owner: Optional[str] = None,
timeout: float = 30.0,
) -> Dict[str, Any]:
"""Same-origin VL self-check: compare BEFORE vs AFTER against the intent.
The browser cannot call the local VL server directly (it sends no CORS
headers), so ``/api/editor/self-check`` proxies here. Returns
``{done, issues, retry_hint, text}``; on any failure (bad input, the model
is unreachable, or a timeout) it degrades to ``done=True`` with
``reason="unavailable"`` so the client never gets stuck retrying.
"""
if not (
isinstance(before, str) and before.startswith("data:image/")
and isinstance(after, str) and after.startswith("data:image/")
):
return {"done": True, "issues": [], "retry_hint": "",
"reason": "unavailable", "text": "DONE"}
bases = local_bases() or list(DEFAULT_LOCAL_BASES)
if not bases:
return {"done": True, "issues": [], "retry_hint": "",
"reason": "unavailable", "text": "DONE"}
base = bases[0]
mdl = await _resolve_model(base, str(model or local_model()), local_model())
messages = [{
"role": "user",
"content": [
{"type": "text", "text": str(prompt or "")},
{"type": "image_url", "image_url": {"url": before}},
{"type": "image_url", "image_url": {"url": after}},
],
}]
text = await _post_chat(base, mdl, messages, None, timeout, max_tokens=140)
if not text:
return {"done": True, "issues": [], "retry_hint": "",
"reason": "unavailable", "text": "DONE"}
out = _parse_self_check(text)
out["model"] = mdl
out["source"] = "local"
return out
def _candidates(owner: Optional[str], override: Optional[str]) -> List[Dict[str, Any]]:
cands: List[Dict[str, Any]] = []
if override:
base, model = _split_override(override)
if model:
cands.append(
{"base": base or local_bases()[0], "model": model, "headers": None, "source": "override"}
)
for base in local_bases():
cands.append({"base": base, "model": local_model(), "headers": None, "source": "local"})
try:
from src.endpoint_resolver import resolve_endpoint
for prefix in ("utility", "default"):
try:
url, model, headers = resolve_endpoint(prefix, owner=owner)
except Exception:
continue
if url and model:
cands.append({"base": url, "model": model, "headers": headers, "source": "hosted"})
break
except Exception:
pass
return cands
async def plan_with_editor_agent(
messages: List[dict],
owner: Optional[str] = None,
override: Optional[str] = None,
timeout: float = 90.0,
response_format: Optional[dict] = None,
) -> Optional[Tuple[str, str, str]]:
"""Return (raw_plan_text, model_used, source) or None.
Tries the local model WITH the screenshot, then the same model text-only
with a VL caption of the screenshot, then the hosted chat model.
``response_format`` (a JSON-schema structured-output constraint) is sent
ONLY to the local editor-agent bases, so a hosted fallback model that does
not support the field never rejects the request.
"""
has_img = _has_image(messages)
text_messages = _strip_images(messages) if has_img else messages
image = _first_image(messages) if has_img else None
caption: Optional[str] = None
_local = {str(b).rstrip("/") for b in local_bases()}
for cand in _candidates(owner, override):
base, model = cand["base"], cand["model"]
headers = cand.get("headers")
is_local = cand["source"] == "local" or str(base).rstrip("/") in _local
rf = response_format if (response_format and is_local) else None
extra = {"response_format": rf} if rf else {}
if has_img:
raw = await _post_chat(base, model, messages, headers, timeout, **extra)
if raw:
return raw, model, cand["source"]
if caption is None and image:
caption = await caption_image(image, owner)
if caption:
raw = await _post_chat(
base, model, _with_caption(text_messages, caption), headers, timeout, **extra,
)
if raw:
return raw, model, cand["source"] + "+caption"
else:
raw = await _post_chat(
base, model, text_messages, headers, timeout, **extra
)
if raw:
return raw, model, cand["source"]
return None
async def list_editor_agent_models(owner: Optional[str] = None) -> List[Dict[str, Any]]:
"""Models available to the editor agent, for the model picker.
Lists the local Qwen3.5-9B (one entry per reachable base) plus the hosted
default/utility chat model that the planner falls back to.
"""
models: List[Dict[str, Any]] = []
seen: set = set()
for base in local_bases():
model = local_model()
key = ("local", base, model)
if key in seen:
continue
seen.add(key)
models.append({
"id": model,
"label": f"{model} · local",
"source": "local",
"vision": True,
"base_url": base,
})
try:
from src.endpoint_resolver import resolve_endpoint
for prefix in ("utility", "default"):
try:
url, model, _headers = resolve_endpoint(prefix, owner=owner)
except Exception:
continue
if url and model:
key = ("hosted", model)
if key not in seen:
seen.add(key)
models.append({
"id": model,
"label": model,
"source": "hosted",
"vision": None,
"base_url": url,
})
break
except Exception:
pass
return models
# ---------------------------------------------------------------------------
# Reachability probe (best effort, non-blocking).
# ---------------------------------------------------------------------------
_PROBE: Dict[str, bool] = {}
def probe_models(base: str, timeout: float = 3.0) -> bool:
try:
with httpx.Client(timeout=timeout) as client:
resp = client.get(str(base).rstrip("/") + "/models")
return resp.status_code == 200
except Exception:
return False
def _probe_worker() -> None:
for base in local_bases():
ok = probe_models(base)
_PROBE[base] = ok
logger.info("editor agent local base %s reachable=%s", base, ok)
cap = caption_base()
if cap:
_PROBE[cap] = probe_models(cap)
logger.info("editor agent caption base %s reachable=%s", cap, _PROBE[cap])
def startup_probe() -> None:
"""Kick off a background reachability probe (never blocks startup)."""
try:
threading.Thread(target=_probe_worker, name="editor-agent-probe", daemon=True).start()
except Exception:
logger.debug("editor agent probe thread failed", exc_info=True)
def probe_status() -> Dict[str, bool]:
return dict(_PROBE)
File diff suppressed because it is too large Load Diff
+26
View File
@@ -68,6 +68,7 @@ def _row(index: int, message: dict[str, Any]) -> dict[str, Any]:
})
return {
"uid": str(message.get("uid") or _uid(index)),
"message_id": str(message.get("message_id") or ""),
"account": str(message.get("account") or "fixture"),
"account_email": str(message.get("account_email") or message.get("to") or ""),
"account_id": str(message.get("account_id") or "fixture"),
@@ -78,6 +79,16 @@ def _row(index: int, message: dict[str, Any]) -> dict[str, Any]:
"folder": str(message.get("folder") or "INBOX"),
"attachments": attachments,
"has_attachments": bool(attachments),
"summary": str(message.get("summary") or message.get("body") or ""),
}
def _account_matches(message: dict[str, Any], account: str | None) -> bool:
selector = str(account or "").strip().casefold()
return not selector or selector in {
str(message.get("account") or "fixture").strip().casefold(),
str(message.get("account_email") or message.get("to") or "").strip().casefold(),
str(message.get("account_id") or "fixture").strip().casefold(),
}
@@ -108,6 +119,12 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
bare = tool.removeprefix("mcp__email__")
if bare == "list_email_accounts":
accounts = {}
for index, message in enumerate(messages):
row = _row(index, message)
identity = (row["account_id"], row["account"], row["account_email"])
accounts.setdefault(identity, {"id": row["account_id"], "name": row["account"],
"email": row["account_email"], "default": not accounts})
return {
"accounts": [{"id": "fixture", "name": "Fixture mailbox", "default": True}],
"output": "Fixture mailbox (default)",
@@ -159,6 +176,7 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
if bare == "read_email":
uid = str(args.get("uid") or "")
message_id = str(args.get("message_id") or "")
folder = str(args.get("folder") or "INBOX")
try:
index = int(uid) - 1
@@ -185,6 +203,7 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
all_messages = _messages()
owner_value = str(owner or "").strip()
visible_index = -1
matches = []
for original in all_messages:
if owner_value and original.get("owner") and original.get("owner") != owner_value:
continue
@@ -193,6 +212,13 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
continue
if not _folder_matches(original.get("folder"), folder):
continue
if not _account_matches(original, args.get("account")):
continue
matches.append(original)
if len(matches) > 1:
return {"error": "Email identity is ambiguous; specify its account and folder.", "exit_code": 1}
if matches:
original = matches[0]
if bare == "archive_email":
original["folder"] = "Archive"
action = "Archived"
+7
View File
@@ -47,6 +47,13 @@ DEFAULT_SETTINGS = {
"image_gen_enabled": False,
"image_model": "",
"image_quality": "medium",
# Local editor-agent/planner model (Quick Edit NL planner). When empty the
# server auto-detects the local Qwen3.5-9B vision server on seneca and falls
# back to the hosted chat model. See src/editor_agent_model.py.
"editor_agent_base_url": "",
"editor_agent_model": "",
"editor_caption_base_url": "",
"editor_caption_model": "",
"vision_model": "",
"vision_enabled": True,
# Ordered fallback chain for the Vision model (image analysis, OCR, tagging).
+34
View File
@@ -0,0 +1,34 @@
"""Process-wide shutdown signalling for graceful restarts.
Server-Sent-Events / streaming responses never complete on their own, so
uvicorn's graceful shutdown would wait for an open browser stream up to the
full ``timeout_graceful_shutdown`` window before exiting. A SIGTERM/SIGINT
handler flips this flag and tells uvicorn to exit; every streaming generator
checks the flag on each loop and ends promptly, so a restart completes in
seconds while ordinary in-flight requests still finish.
"""
from __future__ import annotations
import logging
import threading
logger = logging.getLogger(__name__)
_SHUTTING_DOWN = threading.Event()
def is_shutting_down() -> bool:
"""True once a shutdown (SIGTERM/SIGINT) has been requested."""
return _SHUTTING_DOWN.is_set()
def request_shutdown() -> None:
"""Mark the process as shutting down (idempotent)."""
if not _SHUTTING_DOWN.is_set():
_SHUTTING_DOWN.set()
logger.info("shutdown requested; streaming responses will end promptly")
def reset() -> None:
"""Clear the flag (for tests)."""
_SHUTTING_DOWN.clear()
+14
View File
@@ -216,6 +216,20 @@ _register(
ToolEffect.WRITE_PRIVATE,
result_integrity=ResultIntegrity.EXTERNAL_UNTRUSTED,
)
_register(
{"ai_edit_image"},
ToolEffect.NETWORK_EGRESS,
ToolEffect.WRITE_PRIVATE,
result_integrity=ResultIntegrity.EXTERNAL_UNTRUSTED,
)
_register(
{"get_editor_screenshot"},
ToolEffect.READ_PRIVATE,
)
_register(
{"plan_edit", "run_editor_command"},
ToolEffect.UI_SIDE_EFFECT,
)
_register(
{
"archive_email",
+9 -3
View File
@@ -1558,6 +1558,10 @@ def _agent_subprocess_env() -> dict:
return base
async def _direct_fallback(
tool: str,
content: str,
@@ -2193,7 +2197,8 @@ async def _execute_tool_block_impl(
if tool == "generate_image":
from src.ai_interaction import do_generate_image
desc = "generate_image"
result = await dispatched(do_generate_image(content, session_id=session_id, owner=owner))
image_privacy = (client_runtime_context or {}).get('web_ui_state', {}).get('incognito') is True
result = await dispatched(do_generate_image(content, session_id=session_id, owner=owner, **({'incognito': True} if image_privacy else {})))
elif (tool in NATIVE_FILESYSTEM_TOOLS
and (active_resource_operation() is not None or tool != "apply_patch"
or not _tui_host_bridge_patch_url(client_runtime_context))):
@@ -2353,7 +2358,8 @@ async def _execute_tool_block_impl(
result = await dispatched(do_list_cookbook_servers(content, owner=owner))
elif tool == "edit_image":
desc = "edit_image"
result = await dispatched(do_edit_image(content, owner=owner))
image_privacy = (client_runtime_context or {}).get('web_ui_state', {}).get('incognito') is True
result = await dispatched(do_edit_image(content, owner=owner, **({'incognito': True} if image_privacy else {})))
elif tool == "edit_file":
result = await dispatched(_direct_fallback(tool, content)) or {"error": "edit failed", "exit_code": 1}
desc = result.get("output") or result.get("error") or "edit_file"
@@ -2597,4 +2603,4 @@ def format_tool_result(description: str, result: Dict) -> str:
# External execution bridges are allowed to return canonical result shapes
# without using the built-in tools' output cap. Bound the final model-facing
# text here so one verbose command cannot consume the next request window.
return _truncate("\n".join(parts))
return _truncate("\n".join(parts))
+4
View File
@@ -191,6 +191,10 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = {
"list_cookbook_servers": "List the cookbook's configured servers (remote GPU boxes + local) and which is the current default. Use this BEFORE download_model/serve_model when the user didn't name a host — to decide where to run, or to ask the user which server when ambiguous. Downloads/serves default to the cookbook's selected server, NOT localhost.",
"app_api": "Generic loopback to allowed Odysseus internal endpoints. Use this when the user wants something the UI can do but there's no named tool for it. Covers calendar, gallery, library/documents, memory, notes, tasks, settings, research, compare, cookbook GPUs/state — allowed UI buttons hit /api/* endpoints and you can hit them too. Sensitive auth/user/admin/shell paths and host-control Cookbook mutation routes are blocked; do NOT use app_api for shell commands, package installs, engine rebuilds, or PID signalling. Use named command tooling for shell commands. action='endpoints' with filter=<keyword> lists available endpoints. action='call' takes method+path+body. Hits same routes the UI uses — auth flows free. NOTE: themes are NOT an API endpoint — use the ui_control tool (create_theme / set_theme), not app_api. SESSIONS/CHATS: do NOT use app_api for these — GET /api/sessions returns EMPTY for tool calls (it's owner-filtered and tool calls authenticate as a different identity). EMAIL ACCOUNTS: do NOT use /api/email/accounts via app_api; use list_email_accounts, list_emails, and read_email instead. To list/rename/archive/delete/fork chats use the list_sessions and manage_session tools instead.",
"edit_image": "Edit an image in the gallery: upscale (increase resolution), remove background (rembg), inpaint (fill selected area), or harmonize (blend edits). Specify image ID and action.",
"plan_edit": "Plan a multi-step edit for the open image editor and return ordered editor ops to run.",
"run_editor_command": "Run one deterministic editor op (adjustments, masked filters, selection, layers, import/place, export, screenshot) on the open image editor.",
"get_editor_screenshot": "Capture the open image editor's current canvas as a PNG data URL for inspection.",
"ai_edit_image": "Apply an AI image edit (remove/inpaint objects, generative replace, remove-bg, upscale, denoise, enhance-face) to the open image and place the result as a new layer. Use for object removal/addition/replacement; never approximate those with selection/adjustment/filter ops.",
"trigger_research": "Start a deep research job on any topic — appears in the Deep Research sidebar, streams progress, produces a detailed report. Use for 'research X', 'look into Y', 'do deep research on Z', 'investigate'. NOT a scheduled task — it runs now and surfaces in the sidebar.",
"manage_bg_jobs": "Inspect and control detached background `bash` jobs (the ones started with a `#!bg` marker). action='list' shows this chat's jobs (id/status/age/command); action='output' returns a job's captured output so far (check on a long-running job, or re-read a finished one); action='kill' stops a runaway job by id. Use for 'is the background job done', 'check on that job', 'show the build output', 'kill the background job', 'stop the bg task'. output/kill need a job_id from list.",
}
+83 -3
View File
@@ -8,6 +8,7 @@ Extracted from agent_tools.py to keep schema definitions separate from
tool parsing / execution logic.
"""
import os
import json
import logging
import re
@@ -113,6 +114,15 @@ def _repair_function_arg_aliases(tool_type: str, args: dict[str, Any]) -> dict[s
args["url"] = path
args.pop("path", None)
args.pop("file_path", None)
elif (
tool_type == "ui_control"
and str(args.get("action") or "").casefold() == "open_panel"
and not str(args.get("name") or "").strip()
):
panel = _first_nonempty_string(args, ("panel",))
if panel:
args["name"] = panel
args.pop("panel", None)
if tool_type == "manage_documents" and "limit" not in args and "max_results" in args:
# Collection APIs use both names across the native tool surface. The
# document contract calls this integer ``limit``.
@@ -904,13 +914,13 @@ FUNCTION_TOOL_SCHEMAS = [
"type": "function",
"function": {
"name": "ui_control",
"description": "Control the user interface. Actions: toggle (turn tools on/off), open_panel (open a modal: documents/library, gallery, calendar/schedule, email, sessions, notes, memories/brain, skills, settings, theme, cookbook; calendar supports month/week/year/agenda plus a date; Cookbook supports models/download, launch/serve, active/running, dependencies, and settings views), open_email_reply (legacy UI-only reply opener; prefer email MCP draft_email_reply for assistant-written reply drafts so a normal document-backed email draft is created), set_mode, switch_model, set_theme (built-in presets: dark, light, midnight, cyberpunk, retrowave, forest, ocean, ume, terminal, organs, gpt, claude, cute, eclipse, porcelain, arcade, blueprint, monolith, yoyo), create_theme (CREATE any custom theme with a name + colors object — pick distinctive, evocative hex colors that match the requested aesthetic, NOT generic defaults. The theme auto-applies after creation), get_theme, and get_toggles. When a user asks for ANY theme not in the built-in preset list, ALWAYS use create_theme.",
"description": "Control the user interface. Actions: toggle (turn tools on/off), open_panel (open a modal: documents/library, gallery, calendar/schedule, email, sessions, notes, tasks, memories/brain, skills, settings, theme, cookbook; calendar supports month/week/year/agenda plus a date; Cookbook supports models/download, launch/serve, active/running, dependencies, and settings views), open_email_reply (legacy UI-only reply opener; prefer email MCP draft_email_reply for assistant-written reply drafts so a normal document-backed email draft is created), set_mode, switch_model, set_theme (built-in presets: dark, light, midnight, cyberpunk, retrowave, forest, ocean, ume, terminal, organs, gpt, claude, cute, eclipse, porcelain, arcade, blueprint, monolith, yoyo), create_theme (CREATE any custom theme with a name + colors object — pick distinctive, evocative hex colors that match the requested aesthetic, NOT generic defaults. The theme auto-applies after creation), get_theme, and get_toggles. When a user asks for ANY theme not in the built-in preset list, ALWAYS use create_theme.",
"parameters": {
"type": "object",
"properties": {
"action": {"type": "string", "enum": ["toggle", "open_panel", "open_email_reply", "set_mode", "switch_model", "set_theme", "create_theme", "get_theme", "get_toggles"],
"description": "The UI action. Use set_theme for presets, create_theme to build a custom theme with any hex colors"},
"name": {"type": "string", "description": "For toggle: web, bash, research, incognito, document_editor (aliases: shell, search, deepresearch, documents). For open_panel: documents, gallery, calendar/schedule, email, sessions, notes, brain/memories, skills, settings, theme/themes, cookbook; models and serve are Cookbook-view aliases. For open_email_reply: email UID. For set_theme: a preset theme name. For create_theme: the custom theme name."},
"name": {"type": "string", "description": "For toggle: web, bash, research, incognito, document_editor (aliases: shell, search, deepresearch, documents). For open_panel: documents, gallery, calendar/schedule, email, sessions, notes, tasks, brain/memories, skills, settings, theme/themes, cookbook; models and serve are Cookbook-view aliases. For open_email_reply: email UID. For set_theme: a preset theme name. For create_theme: the custom theme name."},
"view": {"type": "string", "description": "Optional open_panel subview: calendar day/week/month/year/agenda, or Cookbook models/download, launch/serve, active/running, dependencies, settings."},
"value": {"type": "string", "description": "Value: on/off for toggle, agent/chat for set_mode, model name for switch_model, theme name for set_theme, or folder for open_email_reply"},
"uid": {"type": "string", "description": "Email UID for open_email_reply"},
@@ -1510,6 +1520,75 @@ FUNCTION_TOOL_SCHEMAS = [
}
}
},
{
"type": "function",
"function": {
"name": "plan_edit",
"description": "Plan a photo-editing program for the currently open image editor. Returns an ordered list of editor ops to run against the canvas. Route object removal, added content, or replaced content to ai_edit_image/generate_image; use deterministic ops only for tone/colour/filter/crop/text/selection edits. If no image provider is configured for a generative request, say so instead of planning a substitute deterministic edit. Use the canonical adjustment/filter shapes (adjustments.apply/applyMasked params {\"type\", \"params\"}; filters.apply/applyMasked params {\"id\"}); never invent adjustment types or filter ids. New ops: photo.autoTone; retouch.skin/blemish/teeth; selection.face/background. Portrait/look-better ('fix the skin', 'fix my teeth') -> selection.face then retouch.skin (and retouch.teeth for teeth) then ONE gentle photo.autoTone; never a timid single step. Low-key/'fix the lighting' on a dark image: at most ONE global lift (prefer photo.autoTone once or shadows-highlights); never stack autoTone/brightness/exposure/levels, and keep highlights <= 0. Temperature sign: moody/cool/cinematic -> white-balance temperature NEGATIVE; warm/golden/sunset -> POSITIVE. Background softer -> selection.background then filters.applyMasked {id:'blur-gaussian'} (never a plain global blur). A vague prompt with no concrete defect ('clean this up') -> return {\"reply\": <question>, \"steps\": [], \"ask_user\": {\"question\":..., \"options\":[...]}} with 2-3 concrete options instead of guessing.",
"parameters": {
"type": "object",
"properties": {
"prompt": {"type": "string", "description": "What the user wants done to the image"},
"state": {"type": "object", "description": "Optional current editor state (layers, selection, document info)"}
},
"required": ["prompt"]
}
}
},
{
"type": "function",
"function": {
"name": "run_editor_command",
"description": "Run a single deterministic operation on the open image editor and return its result. Use catalog ops such as adjustments.*, filters.applyMasked, selection.*, layers.*, document.importImage, layers.placeImage, export.png, screenshot. Do NOT use for AI/masked edits (use ai_edit_image) or new imagery (use generate_image). Canonical shapes: adjustments.apply/applyMasked params = {\"type\": <brightness-contrast|exposure|white-balance|hue-saturation|vibrance|black-white|shadows-highlights|levels|curves|color-balance|photo-filter|selective-color|gradient-map|invert|posterize|threshold|channel-mixer|color-lookup>, \"params\": {...}}; filters.apply/applyMasked params = {\"id\": <blur-gaussian|blur-motion|blur-zoom|sharpen|sharpen-edges|unsharp-mask|noise-add|noise-despeckle|pixelate-mosaic|pixelate-crystallize|stylize-emboss|stylize-find-edges|stylize-solarize|distort-twirl|distort-pinch|distort-spherize|distort-ripple|render-clouds>}. Never nest a numeric bag under {\"adjustments\": {...}} and never invent adjustment types or filter ids (no \"vignette\" or \"clarity\"). Additional canonical ops: photo.autoTone {}; retouch.skin {smooth,opacity,blemish,radius,threshold}; retouch.blemish {radius,threshold}; retouch.teeth {desat,bright}; selection.face {mode}; selection.background {mode,subject,fuzziness}. At most one global brightness/exposure lift (prefer shadows-highlights; keep highlights <= 0). Never stack global lifts.",
"parameters": {
"type": "object",
"properties": {
"op": {"type": "string", "description": "Editor op name from the catalog, e.g. adjustments.applyMasked"},
"params": {"type": "object", "description": "Parameters for the op"},
"session": {"type": "string", "description": "Editor session id (defaults to the open session)"},
"wait": {"type": "boolean", "description": "Wait for the editor result (default true)"}
},
"required": ["op"]
}
}
},
{
"type": "function",
"function": {
"name": "get_editor_screenshot",
"description": "Capture the open image editor's current canvas as a PNG data URL. Use to inspect the result before choosing the next edit.",
"parameters": {
"type": "object",
"properties": {
"session": {"type": "string", "description": "Editor session id (defaults to the open session)"}
}
}
}
},
{
"type": "function",
"function": {
"name": "ai_edit_image",
"description": "Apply an AI image edit and place the result as a NEW layer, never overwriting the original. Two modes. mode='instruction' (default): send the WHOLE image with a clear prompt and no mask — use a rephrased instruction that NAMES what must stay unchanged, e.g. 'replace the background with a sunny beach, keep the woman exactly the same'. Use this for background/scene replacement, changing the season or style, adding an object to the scene (a moon, a person), and removing diffuse distractions. mode='masked': masked inpaint on a selected region — use ONLY for an explicit regional removal of a clearly identified object ('remove the person on the left'); select the object first so the mask is tight. NEVER approximate object removal/addition/replacement with selection, adjustment, filter or transform ops; if no image provider is configured, report that instead of substituting a deterministic edit. Other ops: remove-bg, upscale (2x/4x), denoise, enhance-face.",
"parameters": {
"type": "object",
"properties": {
"op": {"type": "string", "enum": ["remove-bg", "upscale", "denoise", "enhance-face", "inpaint", "generative"], "description": "AI edit operation"},
"mode": {"type": "string", "enum": ["instruction", "masked"], "description": "instruction = whole-image edit with no mask (background/season/style/add-object/diffuse removal); masked = tight-mask regional removal of a selected object. Default instruction."},
"prompt": {"type": "string", "description": "Instruction for inpaint/generative (in instruction mode, name what must stay unchanged)"},
"image": {"type": "string", "description": "Optional source (data URL or gallery image id); defaults to the current canvas/screenshot"},
"mask": {"type": "string", "description": "Optional base64 PNG mask (white = region to edit); required for mode='masked'"},
"scale": {"type": "number", "description": "Upscale factor (2 or 4)"},
"strength": {"type": "number", "description": "Denoise strength (0-1)"},
"steps": {"type": "integer", "description": "Optional sampler steps. Use 8 for removal/expand edits (default 4 is too weak for those)."},
"place": {"type": "boolean", "description": "Place the result as a new layer (default true)"},
"model": {"type": "string", "description": "Optional image model id to route this edit to (from GET /api/editor/image-models); omit to use the configured default"},
"session": {"type": "string", "description": "Editor session id"}
},
"required": ["op"]
}
}
},
{
"type": "function",
"function": {
@@ -1922,6 +2001,7 @@ FUNCTION_TOOL_SCHEMAS = [
]
# ---------------------------------------------------------------------------
# Converter: native function call -> ToolBlock
# ---------------------------------------------------------------------------
@@ -2380,4 +2460,4 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock
else:
content = json.dumps(args)
return ToolBlock(tool_type, content)
return ToolBlock(tool_type, content)
+2
View File
@@ -143,6 +143,7 @@ PLAN_MODE_READONLY_TOOLS = {
"resolve_contact",
"chat_with_model",
"ask_teacher",
"get_editor_screenshot",
}
@@ -179,6 +180,7 @@ _PLAN_MODE_KNOWN_MUTATORS = {
"download_model", "serve_model",
"stop_served_model", "cancel_download", "adopt_served_model", "serve_preset",
"generate_image", "edit_image", "trigger_research", "manage_research",
"plan_edit", "run_editor_command", "ai_edit_image",
# Shell is never read-only-safe; block it explicitly so it stays out of plan
# mode even if the schema list fails to load.
"bash", "python",
+1
View File
@@ -28,5 +28,6 @@ TOOL_TAGS = {
"tail_serve_output", "list_downloads", "cancel_download", "search_hf_models",
"list_cached_models", "list_serve_presets", "serve_preset", "adopt_served_model",
"list_cookbook_servers", "edit_image", "trigger_research", "manage_research",
"plan_edit", "run_editor_command", "get_editor_screenshot", "ai_edit_image",
"app_api",
} | BUILTIN_EMAIL_TOOLS
+13 -3
View File
@@ -16,7 +16,7 @@ from typing import Dict, Optional
from src.tools._common import _parse_tool_args
async def do_edit_image(content: str, owner: Optional[str] = None) -> Dict:
async def do_edit_image(content: str, owner: Optional[str] = None, *, incognito: bool = False) -> Dict:
"""Create an owner-scoped edited copy of a gallery image."""
try:
args = _parse_tool_args(content)
@@ -45,7 +45,7 @@ async def do_edit_image(content: str, owner: Optional[str] = None) -> Dict:
if not load_settings().get('image_gen_enabled', True):
return {'error': 'Image generation is disabled by the administrator.', 'exit_code': 1}
from src.ai_interaction import do_edit_image as edit_with_model
return await edit_with_model(str(args.get('prompt') or '').strip(), info['path'], owner=owner, size='auto')
return await edit_with_model(str(args.get('prompt') or '').strip(), info['path'], owner=owner, size='auto', incognito=incognito)
from core.database import GalleryImage, SessionLocal
from src.constants import GENERATED_IMAGES_DIR
@@ -79,7 +79,7 @@ async def do_edit_image(content: str, owner: Optional[str] = None) -> Dict:
session_id = source.session_id
db.close()
from src.ai_interaction import do_edit_image as edit_with_model
return await edit_with_model(prompt, str(source_path), session_id=session_id, owner=owner, size='auto')
return await edit_with_model(prompt, str(source_path), session_id=session_id, owner=owner, size='auto', incognito=incognito)
from PIL import Image
@@ -111,6 +111,16 @@ async def do_edit_image(content: str, owner: Optional[str] = None) -> Dict:
output_bytes = output.getvalue()
width, height = image.size
if incognito:
import base64
return {
'output': f'Image edited ({action}). Private result, not saved to gallery.',
'exit_code': 0, 'image_id': None,
'image_url': 'data:image/png;base64,' + base64.b64encode(output_bytes).decode('ascii'),
'image_prompt': source.prompt or action, 'image_model': f'edit_image:{action}',
'image_size': f'{width}x{height}', 'image_quality': source.quality or '',
}
root.mkdir(parents=True, exist_ok=True)
filename = f"{uuid.uuid4().hex[:12]}.png"
(root / filename).write_bytes(output_bytes)
+26 -3
View File
@@ -37,6 +37,7 @@ FAMILY_TOOLS = {
"sessions": frozenset({"list_sessions", "manage_session", "create_session", "send_to_session", "chat_with_model", "pipeline"}),
"image_generation": frozenset({"generate_image"}),
"image_editing": frozenset({"edit_image"}),
"photo_editor": frozenset({"plan_edit", "run_editor_command", "get_editor_screenshot", "ai_edit_image", "generate_image"}),
"transcription": frozenset({"transcribe_media"}),
"media_inspection": frozenset({"inspect_media"}),
"ocr": frozenset({"extract_text"}),
@@ -487,7 +488,8 @@ _BOUND_EDITOR_WRITE = re.compile(
re.I,
)
_BOUND_EDITOR_IMPLICIT_REVISION = re.compile(
r"^\s*" + _REQUEST_PREFIX + r"(?:broaden|expand|deepen|lighten|go\s+deeper|"
r"^\s*" + _REQUEST_PREFIX + r"(?:broaden|expand|deepen|lighten|go\s+deeper|go\s+more\s+into|"
r"g[aå]\s+mer\s+(?:in|ut)\s+p[aå]|"
r"clean\s+(?:this|it|the\s+(?:text|draft|document|doc))\s+up|"
r"give\s+(?:me\s+)?(?:feedback|a\s+critique|suggestions?|comments?)|"
r"remove\b[^.;\n]{0,100}\b(?:mistakes?|errors?|inaccurac(?:y|ies)|misinformation)|"
@@ -633,6 +635,7 @@ _REQUIRED_TOOLS = {
"calendar": "manage_calendar", "notes": "manage_notes",
"tasks": "manage_tasks", "skills": "manage_skills",
"image_generation": "generate_image", "image_editing": "edit_image",
"photo_editor": "plan_edit",
"transcription": "transcribe_media", "media_inspection": "inspect_media", "ocr": "extract_text",
}
@@ -2994,6 +2997,18 @@ def _natural_safe_inventory_operation(message: str, maximum: int | None = None)
return RequiredReadOperation(tool, {"action": action} if action else {}, maximum)
def note_read_intent(user_text: str) -> tuple[str, bool]:
"""Return a quoted mention query and whether the note body is needed."""
text = str(user_text or "")
mention = re.search(r"\bmentions?\s+(['\"])([^'\"]{1,120})\1", text, re.I)
query = mention.group(2).strip() if mention else ""
needs_body = bool(
re.search(r"\b(?:summari[sz]e|summary|main\s+topics)\b", text, re.I)
or re.search(r"\b(?:show|open|read|view)\b.{0,80}\b(?:first|newest|most\s+recent)\s+note\b", text, re.I)
)
return query, needs_body
def required_read_operation_for_request(message: str, history: Iterable = ()) -> RequiredReadOperation | None:
"""Resolve complete list/read requests, or repeat an exact prior read.
@@ -4149,7 +4164,15 @@ def _families_for_tool(tool: str) -> frozenset[str]:
if bare in {"list_sessions", "manage_session", "create_session", "send_to_session",
"chat_with_model", "pipeline"}:
return frozenset({"sessions"})
return frozenset(family for family, tools in FAMILY_TOOLS.items() if bare in tools)
# The photo_editor family is context-only: it is only ever selected by an
# explicit editor request at the route, never inferred from message text.
# Overlapping tools such as generate_image must still resolve to their real
# product family, so exclude photo_editor rather than short-circuiting.
return frozenset(
family
for family, tools in FAMILY_TOOLS.items()
if bare in tools and family != "photo_editor"
)
def _clause_capabilities(text: str) -> set[str]:
@@ -6325,7 +6348,7 @@ def resolve_full_inventory_contract(*, schemas: Iterable[dict], policy: ToolPoli
"""Experimental trained inventory: permissions filter offers; model chooses actions."""
families = frozenset({"calendar", "notes", "tasks", "skills", "memory", "documents",
"email", "search_browser", "shell_files", "cookbook_admin",
"image_editing", "image_generation"})
"image_editing", "image_generation", "photo_editor"})
# ``ui_control`` is the executable bridge for explicit client-interface
# requests (for example, opening the gallery). It is not one of the ten
# persisted-data families, but omitting it here makes the full-inventory
+36
View File
@@ -0,0 +1,36 @@
"""Tests for src/activity.py — in-flight heavy-image-op counters (safe restart)."""
from src import activity
def test_image_op_counter_increments_and_decrements():
assert activity.heavy_image_ops_active() == 0
activity.begin_image_op("a")
activity.begin_image_op("a")
assert activity.heavy_image_ops_active() == 2
activity.end_image_op("a")
assert activity.heavy_image_ops_active() == 1
activity.end_image_op("a")
assert activity.heavy_image_ops_active() == 0
# never goes negative
activity.end_image_op("a")
assert activity.heavy_image_ops_active() == 0
def test_image_op_context_manager_and_busy():
assert activity.busy() is False
with activity.image_op("/api/image/inpaint"):
assert activity.busy() is True
snap = activity.activity()
assert snap["image_ops"] == 1
assert snap["image_ops_by_name"].get("/api/image/inpaint") == 1
assert activity.busy() is False
assert activity.activity()["image_ops"] == 0
def test_image_op_released_on_exception():
try:
with activity.image_op("boom"):
raise RuntimeError("x")
except RuntimeError:
pass
assert activity.heavy_image_ops_active() == 0
File diff suppressed because it is too large Load Diff
+26 -4
View File
@@ -2,6 +2,7 @@ import asyncio
import json
from types import SimpleNamespace
import pytest
from PIL import Image
from src.tools.image import do_edit_image
@@ -122,7 +123,8 @@ def test_edit_image_rejects_actions_without_complete_input_contract():
assert "Use prompt, upscale or rembg" in result["error"]
def test_prompt_edit_forwards_owned_pixels_and_preserves_original(monkeypatch, tmp_path):
@pytest.mark.parametrize('incognito', [False, True])
def test_prompt_edit_forwards_owned_pixels_and_preserves_original(monkeypatch, tmp_path, incognito):
source_path = tmp_path / 'source.png'
Image.new('RGB', (3, 2), 'red').save(source_path)
original = source_path.read_bytes()
@@ -135,9 +137,9 @@ def test_prompt_edit_forwards_owned_pixels_and_preserves_original(monkeypatch, t
return {'image_id': 'edited', 'image_url': '/api/generated-image/edited.png'}
from pathlib import Path
monkeypatch.setattr('src.ai_interaction.do_edit_image', edit)
result = asyncio.run(do_edit_image(json.dumps({'image_id': 'source-id', 'action': 'prompt', 'prompt': 'Add another cow'}), owner='alice'))
result = asyncio.run(do_edit_image(json.dumps({'image_id': 'source-id', 'action': 'prompt', 'prompt': 'Add another cow'}), owner='alice', incognito=incognito))
assert result['image_id'] == 'edited'
assert calls == [('Add another cow', original, {'session_id': 'session-1', 'owner': 'alice', 'size': 'auto'})]
assert calls == [('Add another cow', original, {'session_id': 'session-1', 'owner': 'alice', 'size': 'auto', 'incognito': incognito})]
assert source_path.read_bytes() == original
denied = asyncio.run(do_edit_image('{"image_id":"source-id","action":"prompt","prompt":"edit"}', owner=None))
assert denied['error'] == 'Image not found'
@@ -165,7 +167,7 @@ def test_uploaded_image_edit_uses_owner_scoped_reference(monkeypatch, tmp_path):
monkeypatch.setattr('src.ai_interaction.do_edit_image', edit)
args = '{"image_id":"odysseus://attachment/upload.jpg","action":"prompt","prompt":"Make more realistic"}'
assert asyncio.run(do_edit_image(args, owner='alice')) == {'image_id': 'edited'}
assert calls == [('Make more realistic', str(path), {'owner': 'alice', 'size': 'auto'})]
assert calls == [('Make more realistic', str(path), {'owner': 'alice', 'size': 'auto', 'incognito': False})]
assert resolutions == [('upload.jpg', {'owner': 'alice', 'allow_admin': False})]
assert 'error' in asyncio.run(do_edit_image(args, owner='bob'))
assert len(calls) == 1
@@ -173,3 +175,23 @@ def test_uploaded_image_edit_uses_owner_scoped_reference(monkeypatch, tmp_path):
disabled = asyncio.run(do_edit_image(args, owner='alice'))
assert 'disabled' in disabled['error']
assert len(calls) == 1
def test_private_upscale_does_not_save_image_or_gallery_record(monkeypatch, tmp_path):
source_path = tmp_path / 'source.png'
Image.new('RGB', (3, 2), 'red').save(source_path)
original = source_path.read_bytes()
db = _Db(_source(source_path.name))
monkeypatch.setattr('core.database.SessionLocal', lambda: db)
monkeypatch.setattr('src.constants.GENERATED_IMAGES_DIR', str(tmp_path))
result = asyncio.run(do_edit_image(
'{"image_id":"source-id","action":"upscale","scale":2}',
owner='alice', incognito=True,
))
assert result['exit_code'] == 0
assert result['image_url'].startswith('data:image/png;base64,')
assert not result['image_id']
assert db.added == []
assert not db.committed
assert list(tmp_path.iterdir()) == [source_path]
assert source_path.read_bytes() == original
@@ -0,0 +1,83 @@
"""Server health audit fix: the local editor VLM returned 400s when asked for a
model id it does not serve (the client default 'Qwen3.5-9B-MLX-8bit', or the
stale caption model 'Qwen/Qwen3-VL-30B'). ``_resolve_model`` maps the requested
id to one the base actually serves."""
import asyncio
from src import editor_agent_model as eam
def _cache(base, ids):
eam._MODELS_CACHE[base] = set(ids)
def test_resolve_model_keeps_a_served_request():
base = "http://x:1/v1"
_cache(base, ["Qwen/Qwen3.5-9B", "other"])
assert asyncio.run(eam._resolve_model(base, "Qwen/Qwen3.5-9B", "Qwen/Qwen3.5-9B")) == "Qwen/Qwen3.5-9B"
def test_resolve_model_falls_back_when_requested_is_not_served():
base = "http://x:2/v1"
_cache(base, ["Qwen/Qwen3.5-9B"])
# e.g. the client's MLX build name, or a stale caption model id.
assert asyncio.run(eam._resolve_model(base, "Qwen3.5-9B-MLX-8bit", "Qwen/Qwen3.5-9B")) == "Qwen/Qwen3.5-9B"
assert asyncio.run(eam._resolve_model(base, "Qwen/Qwen3-VL-30B", "Qwen/Qwen3.5-9B")) == "Qwen/Qwen3.5-9B"
def test_resolve_model_picks_a_served_id_when_fallback_absent():
base = "http://x:3/v1"
_cache(base, ["B", "A"])
assert asyncio.run(eam._resolve_model(base, "nope", "also-nope")) == "A"
def test_resolve_model_keeps_request_when_served_list_unknown():
base = "http://x:4/v1"
_cache(base, [])
# Server unreachable / no models listed -> don't rewrite the request.
assert asyncio.run(eam._resolve_model(base, "some/model", "fallback")) == "some/model"
def test_self_check_resolves_the_model_before_calling(monkeypatch):
monkeypatch.setattr(eam, "local_bases", lambda: ["http://editor.test/v1"])
base = eam.local_bases()[0]
_cache(base, ["Qwen/Qwen3.5-9B"])
seen = {}
async def fake_post(b, model, messages, headers, timeout, **kwargs):
seen["model"] = model
return "DONE"
monkeypatch.setattr(eam, "_post_chat", fake_post)
out = asyncio.run(eam.self_check(
"data:image/png;base64,AAAA", "data:image/png;base64,BBBB",
"make it brighter", model="Qwen3.5-9B-MLX-8bit"))
assert seen["model"] == "Qwen/Qwen3.5-9B"
assert out["done"] is True
def test_caption_resolves_the_caption_model(monkeypatch):
monkeypatch.setattr(eam, "caption_base", lambda: "http://caption.test/v1")
seen = {}
async def fake_post(b, model, messages, headers, timeout, **kwargs):
seen["base"] = b
seen["model"] = model
return "a photo"
monkeypatch.setattr(eam, "_post_chat", fake_post)
import src.editor_agent_model as m
_cache(m.caption_base(), ["Qwen/Qwen3.5-9B"])
txt = asyncio.run(m.caption_image("data:image/png;base64,AAAA"))
assert txt == "a photo"
assert seen["model"] == "Qwen/Qwen3.5-9B"
def test_unconfigured_editor_helpers_do_not_probe_private_hosts(monkeypatch):
monkeypatch.setattr(eam, "local_bases", lambda: [])
monkeypatch.setattr(eam, "caption_base", lambda: "")
monkeypatch.setattr(eam, "probe_models", lambda *a: (_ for _ in ()).throw(AssertionError("unexpected probe")))
eam._probe_worker()
assert asyncio.run(eam.caption_image("data:image/png;base64,AAAA")) is None
result = asyncio.run(eam.self_check("data:image/png;base64,AAAA", "data:image/png;base64,BBBB", "brighter"))
assert result["reason"] == "unavailable"
+171
View File
@@ -0,0 +1,171 @@
"""Planner routing regression suite (HARNESS-owned).
Turns PHOTOSHOP's bench prompt families (vague, restyle, bg-remove/scene,
tone/auto, generative, structural) into assertions on the ROUTE CLASS the
server planner picks, so a future planner change cannot silently re-break
routing. The classes are:
ask_user -> the planner is unsure; it must ask with 2-3 choices and
apply NO operations (never a silent tone default)
deterministic -> a deterministic editor op / recipe (adjustments, filters,
crop, text, rotate, fill, recolour ...)
ai_edit_image -> a generative image-model edit (content add/remove/replace,
restyle, look)
generate_image -> text-to-image for a brand-new image
These use the pure routing helpers (``_plan_special`` / ``_generative_kind`` /
``_fallback_plan``), so no model or network call is made.
"""
import json
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from routes import editor_agent_routes as eaar
_STATE = {"width": 900, "height": 600}
_PROVIDER = True
@pytest.fixture(autouse=True)
def _reset_planner():
eaar.set_editor_planner(None)
yield
eaar.set_editor_planner(None)
@pytest.fixture()
def client():
app = FastAPI()
app.include_router(eaar.setup_editor_agent_routes())
return TestClient(app)
def _classify(prompt, provider=_PROVIDER):
"""Return (route_class, ops) for a prompt using the deterministic helpers."""
special = eaar._plan_special(prompt, _STATE, provider)
if special:
steps = list(special[0])
else:
fallback = eaar._fallback_plan(prompt)
if fallback.get("ask_user"):
return "ask_user", []
steps = list(fallback.get("steps") or [])
ops = [str(s.get("op") or "") for s in steps]
if "generate_image" in ops:
return "generate_image", ops
if "ai_edit_image" in ops:
return "ai_edit_image", ops
return "deterministic", ops
# (prompt, expected route class)
BENCH_ROUTING = [
# vague -> ask (never a silent tone tweak)
("do the thing", "ask_user"),
("give it some oomph", "ask_user"),
("make it better", "ask_user"),
# whole-image restyle -> model edit
("make it look like an oil painting", "ai_edit_image"),
("turn it into an anime", "ai_edit_image"),
("make it a comic book illustration", "ai_edit_image"),
("make it look like cyberpunk", "ai_edit_image"),
# background / scene replacement -> model edit
("put her on a beach", "ai_edit_image"),
("make the background nicer", "ai_edit_image"),
("make it snowy", "ai_edit_image"),
# generative content add / remove -> model edit
("add a full moon in the sky", "ai_edit_image"),
("remove the distractions", "ai_edit_image"),
("remove the person on the left", "ai_edit_image"),
# brand-new image -> text-to-image
("create a new image of a full moon", "generate_image"),
("generate an image of a sunset", "generate_image"),
("make a logo", "generate_image"),
# tone / auto / look / structural -> deterministic
("make it black and white", "deterministic"),
("make it pop", "deterministic"),
("make it look professional", "deterministic"),
("make only the red car blue", "deterministic"),
("make a 1080x1350 Instagram post from this", "deterministic"),
("add a title TOKYO with a drop shadow", "deterministic"),
("extend the canvas to 16:9", "deterministic"),
("save as PSD with layers", "deterministic"),
("fill the white area with red", "deterministic"),
("rotate the whole image 90 degrees", "deterministic"),
]
@pytest.mark.parametrize("prompt,expected", BENCH_ROUTING)
def test_bench_prompts_route_to_expected_class(prompt, expected):
route_class, ops = _classify(prompt)
assert route_class == expected, f"{prompt!r} -> {route_class} {ops} (want {expected})"
def test_ask_plans_offer_two_to_three_choices():
for prompt in ("do the thing", "give it some oomph", "make it better"):
plan = eaar._fallback_plan(prompt)
ask = plan.get("ask_user")
assert isinstance(ask, dict), prompt
opts = ask.get("options") or []
assert 2 <= len(opts) <= 3, (prompt, opts)
assert plan.get("steps") == []
def test_object_removal_uses_selection_plus_model_edit():
_cls, ops = _classify("remove the person on the left")
assert "ai_edit_image" in ops
assert any(op.startswith("selection.") for op in ops), ops
def test_recolour_never_uses_a_flat_fill_or_a_global_tint():
# Every recolour selects the object by description and either shifts the
# masked hue or asks the image model for the masked region — never a flat
# selection.fill and never a whole-image adjustments.apply.
for prompt in ("make only the red car blue", "make the red car white", "make the flowers bright pink"):
_cls, ops = _classify(prompt)
assert "selection.samText" in ops, (prompt, ops)
assert "selection.fill" not in ops, (prompt, ops)
assert "adjustments.apply" not in ops, (prompt, ops)
def test_chromatic_recolour_is_a_deterministic_masked_hue_shift():
cls, ops = _classify("make only the red car blue")
assert cls == "deterministic", ops
assert ops == ["selection.samText", "adjustments.applyMasked"], ops
def test_new_image_request_from_scratch_vs_edit_of_current_photo():
# A photo edit ("make it a <style>") must NOT be treated as a new image.
assert eaar._generative_kind("make it a comic book illustration") == "restyle"
assert eaar._is_new_image_request("make it a comic book illustration") is False
# A from-scratch generation still is.
assert eaar._is_new_image_request("create an illustration of a cat") is True
assert eaar._generative_kind("create an illustration of a cat") == "add"
def test_plan_route_returns_deterministic_recolour(client):
eaar.set_editor_planner(lambda messages, owner: json.dumps({"reply": "ok", "steps": []}))
r = client.post("/api/editor/plan", json={"prompt": "make only the red car blue", "state": _STATE})
assert r.status_code == 200
data = r.json()
assert data["planner"] == "deterministic"
assert [s["op"] for s in data["steps"]] == ["selection.samText", "adjustments.applyMasked"]
def test_plan_route_asks_for_a_vague_prompt(client):
eaar.set_editor_planner(lambda messages, owner: json.dumps({"reply": "ok", "steps": []}))
r = client.post("/api/editor/plan", json={"prompt": "do the thing", "state": _STATE})
assert r.status_code == 200
data = r.json()
assert data.get("ask_user"), data
assert data.get("steps") == []
def test_plan_route_routes_restyle_to_model_edit(client):
eaar.set_editor_planner(lambda messages, owner: json.dumps({"reply": "ok", "steps": []}))
r = client.post("/api/editor/plan", json={"prompt": "turn it into an anime", "state": _STATE})
assert r.status_code == 200
ops = [s["op"] for s in r.json()["steps"]]
assert "ai_edit_image" in ops, ops
@@ -11,6 +11,7 @@ from src.clean_agent_preview import (
from src.turn_contract import (
editor_request_instructions, requested_capabilities, selected_tools_for_request,
preserve_bound_editor_selected_tools, resolve_turn_contract,
targets_bound_editor_request,
)
from src.tool_schemas import FUNCTION_TOOL_SCHEMAS
from src.tool_policy import ToolPolicy
@@ -72,6 +73,16 @@ def test_direct_edits_share_the_capability_router(prompt):
assert not active_editor_suggestion_request(doc, prompt)
@pytest.mark.parametrize('prompt', [
'ga mer ut pa att vi tanker pa framtid for bjorn och att vi faktist trivs sa bra har',
'Go more into how we are thinking about the future.',
])
def test_implicit_revision_followups_route_to_active_document_only(prompt):
assert requested_capabilities(prompt, active_document=True) == {'documents'}
assert requested_capabilities(prompt, active_document=False) == frozenset()
assert targets_bound_editor_request(prompt)
def test_source_review_still_requires_tool_completion_after_research():
schemas = [s for s in FUNCTION_TOOL_SCHEMAS if s['function']['name'] in {'web_search', 'suggest_document'}]
assert required_active_editor_tool_choice(active_editor_target=True, suggestion_target=True,
@@ -0,0 +1,79 @@
"""Email list stale-while-revalidate (SWR) regression tests.
A plain list view must never block ~2-5 s on IMAP while we already hold a
recent snapshot or a durable index: it should return the snapshot immediately
(with ``sync.stale``) and refresh in the background. Blocking IMAP only remains
for a genuine cold start (no snapshot AND no indexed rows).
"""
import asyncio
import pathlib
from routes.email import email_routes as er
SRC = pathlib.Path(er.__file__).read_text(encoding="utf-8")
def _list_endpoint():
router = er.setup_email_routes()
for route in router.routes:
if getattr(route, "name", "") == "list_emails":
return route.endpoint
raise AssertionError("list_emails route not found")
def _call(endpoint, **over):
kwargs = dict(
folder="INBOX", limit=50, offset=0, filter="all", from_addr=None,
account_id=None, has_attachments=0, date_from=None, date_to=None,
cached_only=0, cache_bust=None, refresh=0, owner="pewds",
)
kwargs.update(over)
return asyncio.run(endpoint(**kwargs))
def test_list_returns_index_snapshot_with_a_stale_marker_without_blocking_on_imap(monkeypatch):
endpoint = _list_endpoint()
monkeypatch.setattr(
er, "_email_index_list",
lambda owner, account_id, folder, filter_, limit, offset, has_att: (
[{"uid": "11", "subject": "Indexed", "is_read": False}], 1, "2026-10-11T00:00:00Z",
),
)
# The synchronous path must never open IMAP; if it did the request would
# raise instead of returning the index snapshot immediately.
monkeypatch.setattr(er, "_imap_connect", lambda *a, **k: (_ for _ in ()).throw(
AssertionError("SWR must not open IMAP when an index snapshot exists")))
result = _call(endpoint, cache_bust="123")
assert result["total"] == 1
assert [e["uid"] for e in result["emails"]] == ["11"]
assert result["sync"]["stale"] is True
assert result["sync"]["source"] in ("index", "cache_stale")
def test_cold_start_with_no_snapshot_and_no_index_is_not_marked_stale(monkeypatch):
# With neither a snapshot nor an index, SWR must NOT fire: the request
# falls through to the real (blocking) path, so the response is not a
# stale-marked snapshot.
endpoint = _list_endpoint()
monkeypatch.setattr(er, "_email_index_list", lambda *a, **k: ([], 0, None))
monkeypatch.setattr(er, "_imap_connect", lambda *a, **k: (_ for _ in ()).throw(
RuntimeError("offline test")))
try:
result = _call(endpoint)
except Exception:
return # the cold path legitimately attempted real IMAP
assert isinstance(result, dict)
assert not (result.get("sync") or {}).get("stale")
def test_swr_contract_is_wired_into_the_list_path():
assert "_LIST_STALE" in SRC
assert "def _list_stale_put" in SRC
assert "def _schedule_list_refresh" in SRC
assert 'sync_meta["source"] = "cache_stale"' in SRC
assert 'sync_meta["stale"] = True' in SRC
# The manual-refresh contract the UI relies on must stay intact.
assert "manual_refresh = bool(refresh)" in SRC
assert "_invalidate_list_cache(account_id, folder)" in SRC
assert "cached_rows = {} if refresh else _email_index_rows" in SRC
+155
View File
@@ -76,3 +76,158 @@ def test_mixed_endpoint_requires_configured_image_model(monkeypatch):
assert lookup(Query(), endpoint.base_url, 'alice', 'text-model') is None
assert lookup(Query(), endpoint.base_url, 'alice', 'openai/gpt-image-1') is None
assert lookup(Query(), 'https://unregistered.test/v1', 'alice', 'openai/gpt-5-image') is None
def _form_fields(content, content_type):
"""Parse simple multipart form fields (text parts only) from a request body."""
boundary = content_type.split('boundary=')[1].strip().strip('"')
fields = {}
for part in content.split(('--' + boundary).encode()):
if b'name="' not in part:
continue
head, _, body = part.partition(b'\r\n\r\n')
import re as _re
match = _re.search(rb'name="([^"]+)"', head)
if not match:
continue
fields[match.group(1).decode()] = body.rstrip(b'\r\n-').decode('latin1')
return fields
def _self_hosted_inpaint(monkeypatch, image, mask, body_extra, respond, include_mask=True):
base_url = 'http://100.69.120.65:8188/v1'
endpoint = SimpleNamespace(base_url=base_url, api_key='local', owner='alice')
monkeypatch.setattr(gallery, 'SessionLocal', lambda: SimpleNamespace(close=lambda: None))
monkeypatch.setattr(gallery, 'require_privilege', lambda *a: 'alice')
monkeypatch.setattr(gallery, '_visible_image_endpoint_for_base', lambda *a: endpoint)
monkeypatch.setattr('src.url_safety.check_outbound_url', lambda *a, **k: (True, ''))
_real_client = httpx.AsyncClient
monkeypatch.setattr(httpx, 'AsyncClient',
lambda **kw: _real_client(transport=httpx.MockTransport(respond), **kw))
class Request:
async def json(self):
payload = {'_endpoint': base_url, 'image': png(image), 'prompt': 'do a thing'}
if include_mask:
payload['mask'] = png(mask)
payload.update(body_extra)
return payload
route = next(r for r in gallery.setup_gallery_routes().routes if r.path == '/api/image/inpaint')
return asyncio.run(route.endpoint(Request()))
def test_self_hosted_inpaint_caps_oversize_and_restores_dimensions(monkeypatch):
seen = {}
def respond(request):
assert str(request.url) == 'http://100.69.120.65:8188/v1/images/edits'
seen.update(_form_fields(request.content, request.headers['content-type']))
return httpx.Response(200, json={'data': [{'b64_json': png(Image.new('RGB', (768, 576), 'blue'))}]})
out = _self_hosted_inpaint(monkeypatch, Image.new('RGB', (1600, 1200), 'red'),
Image.new('L', (1600, 1200), 255),
{'steps': 8}, respond)
# Working size is capped to a model-friendly size (<=768 long side, /16).
assert seen['size'] == '768x576'
# A requested step budget passes through up to the cap (default 8).
assert seen['steps'] == '8'
# The returned image keeps the original dimensions.
assert Image.open(io.BytesIO(base64.b64decode(out['image']))).size == (1600, 1200)
def test_self_hosted_inpaint_clamps_steps_to_the_cap(monkeypatch):
seen = {}
def respond(request):
seen.update(_form_fields(request.content, request.headers['content-type']))
return httpx.Response(200, json={'data': [{'b64_json': png(Image.new('RGB', (640, 480), 'blue'))}]})
_self_hosted_inpaint(monkeypatch, Image.new('RGB', (640, 480), 'red'),
Image.new('L', (640, 480), 255), {'steps': 20}, respond)
# Over-cap requests are clamped to the configured maximum (default 8).
assert seen['steps'] == '8'
def test_self_hosted_inpaint_leaves_small_images_unchanged(monkeypatch):
seen = {}
def respond(request):
seen.update(_form_fields(request.content, request.headers['content-type']))
return httpx.Response(200, json={'data': [{'b64_json': png(Image.new('RGB', (640, 480), 'green'))}]})
out = _self_hosted_inpaint(monkeypatch, Image.new('RGB', (640, 480), 'red'),
Image.new('L', (640, 480), 255), {}, respond)
# Under the cap: dimensions pass through untouched; with no requested count
# sent the moderate default (4) is used.
assert seen['size'] == '640x480'
assert seen['steps'] == '4'
assert Image.open(io.BytesIO(base64.b64decode(out['image']))).size == (640, 480)
def test_self_hosted_inpaint_without_mask_uses_a_full_white_mask(monkeypatch):
"""A whole-image instruction edit may omit the mask; it must not raise
KeyError -> 400 (the health-audit regression)."""
seen = {}
mask_bytes = {}
def respond(request):
seen.update(_form_fields(request.content, request.headers['content-type']))
# The provider must receive a (full-white) mask, not a missing one.
import re as _re
parts = request.content.split(b'--')
for part in parts:
if b'name="mask"' in part:
head, _, body = part.partition(b'\r\n\r\n')
mask_bytes['png'] = body.rstrip(b'\r\n-')
return httpx.Response(200, json={'data': [{'b64_json': png(Image.new('RGB', (32, 32), 'blue'))}]})
out = _self_hosted_inpaint(monkeypatch, Image.new('RGB', (32, 32), 'red'),
None, {}, respond, include_mask=False)
assert 'image' in out, out
assert seen.get('size') == '32x32'
# The synthesized mask is image-sized and all-white (whole image editable).
mask_img = Image.open(io.BytesIO(mask_bytes['png'])).convert('L')
assert mask_img.size == (32, 32)
assert mask_img.getextrema() == (255, 255)
def test_openai_inpaint_without_mask_is_a_clean_400(monkeypatch):
"""OpenAI edits need a mask; a missing one must be a clean 400, not a
KeyError traceback."""
from fastapi import HTTPException
endpoint = SimpleNamespace(base_url='https://api.openai.com/v1', api_key='sk-x', owner='alice')
monkeypatch.setattr(gallery, 'SessionLocal', lambda: SimpleNamespace(close=lambda: None))
monkeypatch.setattr(gallery, 'require_privilege', lambda *a: 'alice')
monkeypatch.setattr(gallery, '_visible_image_endpoint_for_base', lambda *a: endpoint)
monkeypatch.setattr('src.url_safety.check_outbound_url', lambda *a, **k: (True, ''))
class Request:
async def json(self):
return {'_endpoint': 'https://api.openai.com/v1',
'image': png(Image.new('RGB', (32, 32), 'red')), 'prompt': 'x'}
route = next(r for r in gallery.setup_gallery_routes().routes if r.path == '/api/image/inpaint')
try:
asyncio.run(route.endpoint(Request()))
raise AssertionError('expected a 400 for a missing mask')
except HTTPException as exc:
assert exc.status_code == 400
assert 'mask' in str(exc.detail).lower()
def test_self_hosted_inpaint_forwards_request_id(monkeypatch):
"""A client-supplied request_id is forwarded to the provider so the client
can poll GET /api/image/edit-progress for step progress."""
seen = {}
def respond(request):
seen.update(_form_fields(request.content, request.headers.get('content-type', '')))
return httpx.Response(200, json={'model': 'qwen-image-edit-2511',
'data': [{'b64_json': png(Image.new('RGBA', (32, 32), 'red'))}]})
out = _self_hosted_inpaint(monkeypatch, Image.new('RGB', (32, 32), 'red'),
Image.new('L', (32, 32), 255), {'request_id': 'job-777'}, respond)
assert 'image' in out, out
assert seen.get('request_id') == 'job-777'
+120
View File
@@ -0,0 +1,120 @@
"""Tests for the image-op progress contract (HARNESS-owned).
The 7011 side accepts an optional client-supplied ``request_id`` on heavy image
ops and exposes ``GET /api/image/edit-progress`` which proxies the provider's
``/v1/images/progress/{request_id}`` route. When the provider has no such route
the endpoint returns ``{"available": false}`` so the client keeps an
indeterminate bar.
"""
import asyncio
from types import SimpleNamespace
import httpx
import routes.gallery_routes as gallery
def _route(path):
return next(r for r in gallery.setup_gallery_routes().routes if r.path == path)
class _Req:
def __init__(self, query):
self.query_params = query
def _fake_db():
class Db:
def close(self):
pass
def query(self, *args, **kwargs):
return self
def filter(self, *args, **kwargs):
return self
def first(self):
return None
return Db()
def test_edit_progress_requires_request_id(monkeypatch):
monkeypatch.setattr(gallery, "require_privilege", lambda *a: "alice")
route = _route("/api/image/edit-progress")
try:
asyncio.run(route.endpoint(_Req({})))
except Exception as exc: # FastAPI HTTPException
assert getattr(exc, "status_code", None) == 400
else:
raise AssertionError("expected a 400 for a missing request_id")
def test_edit_progress_rejects_an_unsafe_request_id(monkeypatch):
monkeypatch.setattr(gallery, "require_privilege", lambda *a: "alice")
monkeypatch.setattr(gallery, "SessionLocal", lambda: _fake_db())
route = _route("/api/image/edit-progress")
try:
asyncio.run(route.endpoint(_Req({"request_id": "../secret"})))
except Exception as exc:
assert getattr(exc, "status_code", None) == 400
else:
raise AssertionError("expected a 400 for an unsafe request_id")
def test_edit_progress_returns_provider_payload(monkeypatch):
endpoint = SimpleNamespace(base_url="http://100.69.120.65:8188/v1", api_key="k")
monkeypatch.setattr(gallery, "SessionLocal", lambda: _fake_db())
monkeypatch.setattr(gallery, "require_privilege", lambda *a: "alice")
monkeypatch.setattr(gallery, "_first_visible_image_endpoint", lambda *a: endpoint)
seen = {}
def respond(request):
seen["url"] = str(request.url)
# Live seneca contract stages: loading | editing | compositing | done | error.
return httpx.Response(
200,
json={"stage": "editing", "step": 3, "steps": 8, "eta_s": 42.0, "progress": 0.375},
)
client = httpx.AsyncClient
monkeypatch.setattr(httpx, "AsyncClient", lambda **k: client(transport=httpx.MockTransport(respond), **k))
out = asyncio.run(_route("/api/image/edit-progress").endpoint(_Req({"request_id": "abc-123"})))
assert out["available"] is True
assert out["stage"] == "editing" # passed through unchanged
assert out["step"] == 3 and out["steps"] == 8
assert seen["url"].endswith("/images/progress/abc-123")
def test_edit_progress_available_false_when_provider_has_no_route(monkeypatch):
endpoint = SimpleNamespace(base_url="http://100.69.120.65:8188/v1", api_key="k")
monkeypatch.setattr(gallery, "SessionLocal", lambda: _fake_db())
monkeypatch.setattr(gallery, "require_privilege", lambda *a: "alice")
monkeypatch.setattr(gallery, "_first_visible_image_endpoint", lambda *a: endpoint)
client = httpx.AsyncClient
monkeypatch.setattr(
httpx, "AsyncClient",
lambda **k: client(transport=httpx.MockTransport(lambda req: httpx.Response(404)), **k),
)
out = asyncio.run(_route("/api/image/edit-progress").endpoint(_Req({"request_id": "abc"})))
assert out["available"] is False
def test_edit_progress_available_false_when_unreachable(monkeypatch):
endpoint = SimpleNamespace(base_url="http://100.69.120.65:8188/v1", api_key="k")
monkeypatch.setattr(gallery, "SessionLocal", lambda: _fake_db())
monkeypatch.setattr(gallery, "require_privilege", lambda *a: "alice")
monkeypatch.setattr(gallery, "_first_visible_image_endpoint", lambda *a: endpoint)
class Boom:
async def __aenter__(self):
raise httpx.ConnectError("nope")
async def __aexit__(self, *a):
return False
monkeypatch.setattr(httpx, "AsyncClient", lambda **k: Boom())
out = asyncio.run(_route("/api/image/edit-progress").endpoint(_Req({"request_id": "abc"})))
assert out["available"] is False
+201
View File
@@ -0,0 +1,201 @@
"""Heavy in-process image ops must not load GB-scale models by default.
remove-bg / SAM mask / Real-ESRGAN upscale+denoise used to load rembg / torch
models inside the web process and OOM-killed the host. They now default to a
graceful 503, proxy to the seneca remote ops service when
``ODYSSEUS_IMAGE_OPS_BASE`` is set (translate to the /v1/images/* contract), and
only run in-process when explicitly enabled via ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS.
"""
import asyncio
import httpx
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from routes.gallery import gallery_routes as gr
HEAVY_ROUTES = [
("/api/image/remove-bg", {"image": "x"}, "Background removal"),
("/api/image/mask", {"image": "x"}, "Smart mask"),
("/api/image/upscale-local", {"image": "x"}, "AI upscale"),
("/api/image/denoise", {"image": "x"}, "AI denoise"),
]
@pytest.fixture(autouse=True)
def _clean_env(monkeypatch):
monkeypatch.setattr(gr, "require_privilege", lambda *a, **k: None)
monkeypatch.delenv("ODYSSEUS_IMAGE_OPS_BASE", raising=False)
monkeypatch.delenv("ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS", raising=False)
@pytest.fixture()
def client():
app = FastAPI()
app.include_router(gr.setup_gallery_routes())
return TestClient(app)
def test_local_image_models_disabled_by_default():
assert gr._local_image_models_enabled() is False
@pytest.mark.parametrize("value", ["1", "true", "Yes", "ON"])
def test_local_image_models_enabled_env(monkeypatch, value):
monkeypatch.setenv("ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS", value)
assert gr._local_image_models_enabled() is True
@pytest.mark.parametrize("value", ["", "0", "no", "off", "maybe"])
def test_local_image_models_disabled_env(monkeypatch, value):
monkeypatch.setenv("ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS", value)
assert gr._local_image_models_enabled() is False
def test_rss_mb_is_positive():
assert gr._rss_mb() > 0
def test_route_heavy_image_op_returns_none_when_enabled(monkeypatch):
monkeypatch.setenv("ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS", "1")
assert asyncio.run(gr._route_heavy_image_op("/api/image/denoise", {}, label="x")) is None
@pytest.mark.parametrize("path,body,label", HEAVY_ROUTES)
def test_heavy_ops_503_when_disabled(client, path, body, label):
resp = client.post(path, json=body)
assert resp.status_code == 503
detail = resp.json().get("detail", "")
assert "temporarily unavailable" in detail
assert label in detail
def test_remove_bg_disabled_does_not_import_rembg(monkeypatch):
import sys
monkeypatch.setitem(sys.modules, "rembg", None)
app = FastAPI()
app.include_router(gr.setup_gallery_routes())
client = TestClient(app)
resp = client.post("/api/image/remove-bg", json={"image": "x"})
assert resp.status_code == 503
class _FakeResponse:
def __init__(self, payload, status_code=200):
self._payload = payload
self.status_code = status_code
def json(self):
return self._payload
class _FakeAsyncClient:
calls = []
response = _FakeResponse({})
def __init__(self, *args, **kwargs):
pass
async def __aenter__(self):
return self
async def __aexit__(self, *args):
return False
async def post(self, url, json=None, files=None, data=None):
_FakeAsyncClient.calls.append((url, {"json": json, "files": files, "data": data}))
return _FakeAsyncClient.response
def _client():
app = FastAPI()
app.include_router(gr.setup_gallery_routes())
return TestClient(app)
def test_remove_bg_proxies_to_remote_contract(monkeypatch):
monkeypatch.setenv("ODYSSEUS_IMAGE_OPS_BASE", "http://seneca.test:8188")
monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient)
_FakeAsyncClient.calls = []
_FakeAsyncClient.response = _FakeResponse(
{"data": [{"b64_json": "RB", "mask_b64": "M", "bbox": [0, 0, 1, 1]}], "model": "birefnet"}
)
resp = _client().post("/api/image/remove-bg", json={"image": "AAAA"})
assert resp.status_code == 200
assert resp.json()["image"] == "RB"
url, kw = _FakeAsyncClient.calls[-1]
assert url == "http://seneca.test:8188/v1/images/remove-background"
assert "image" in kw["files"]
def test_mask_proxies_to_remote_contract(monkeypatch):
monkeypatch.setenv("ODYSSEUS_IMAGE_OPS_BASE", "http://seneca.test:8188")
monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient)
_FakeAsyncClient.calls = []
_FakeAsyncClient.response = _FakeResponse(
{"masks": [{"b64_json": "MASK", "bbox": [10, 20, 30, 40], "score": 0.9, "label": "person"}],
"model": "sam+grounding"}
)
resp = _client().post("/api/image/mask", json={"image": "AAAA", "text": "person"})
assert resp.status_code == 200
body = resp.json()
assert body["mask"] == "MASK"
# remote [x,y,w,h] -> app [x1,y1,x2,y2]
assert body["bbox"] == [10, 20, 40, 60]
url, kw = _FakeAsyncClient.calls[-1]
assert url == "http://seneca.test:8188/v1/images/mask"
assert kw["data"]["prompt"] == "person"
def test_mask_picks_top_score_not_first(monkeypatch):
monkeypatch.setenv("ODYSSEUS_IMAGE_OPS_BASE", "http://seneca.test:8188")
monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient)
_FakeAsyncClient.response = _FakeResponse(
{"masks": [
{"b64_json": "LOW", "bbox": [0, 0, 1, 1], "score": 0.2},
{"b64_json": "HIGH", "bbox": [0, 0, 2, 2], "score": 0.95},
{"b64_json": "MID", "bbox": [0, 0, 3, 3], "score": 0.5},
], "model": "sam+grounding"}
)
resp = _client().post("/api/image/mask", json={"image": "AAAA", "text": "person"})
assert resp.status_code == 200
assert resp.json()["mask"] == "HIGH"
def test_mask_box_is_sent_as_xywh(monkeypatch):
monkeypatch.setenv("ODYSSEUS_IMAGE_OPS_BASE", "http://seneca.test:8188")
monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient)
_FakeAsyncClient.calls = []
_FakeAsyncClient.response = _FakeResponse({"masks": [{"b64_json": "M", "bbox": [0, 0, 1, 1], "score": 1.0}]})
resp = _client().post("/api/image/mask", json={"image": "AAAA", "box": [10, 20, 40, 60]})
assert resp.status_code == 200
_, kw = _FakeAsyncClient.calls[-1]
assert kw["data"]["box"] == "10,20,30,40"
def test_mask_remote_empty_masks_is_200_nothing_selected(monkeypatch):
monkeypatch.setenv("ODYSSEUS_IMAGE_OPS_BASE", "http://seneca.test:8188")
monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient)
_FakeAsyncClient.response = _FakeResponse({"masks": [], "model": "sam+grounding"})
resp = _client().post("/api/image/mask", json={"image": "AAAA", "text": "person"})
assert resp.status_code == 200
body = resp.json()
assert body["mask"] is None
assert body["masks"] == []
assert "Nothing selected" in body["error"]
def test_remote_proxy_surfaces_error_status(monkeypatch):
monkeypatch.setenv("ODYSSEUS_IMAGE_OPS_BASE", "http://seneca.test:8188")
monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient)
_FakeAsyncClient.response = _FakeResponse({}, status_code=500)
resp = _client().post("/api/image/remove-bg", json={"image": "AAAA"})
assert resp.status_code == 503
assert "temporarily unavailable" in resp.json().get("detail", "")
@@ -0,0 +1,124 @@
"""Image endpoint fallback: image models on mixed (non image-only) endpoints.
Chat image generation resolves its model by name across every enabled endpoint,
so a ``qwen-image-edit`` / ``gpt-image-1`` model registered on a shared
OpenAI-compatible gateway works there. The editor's generative ops used a
stricter image-only lookup and reported "No image generation endpoint
configured" on those same setups. These tests pin the fallback that makes the
editor find the same endpoint.
"""
from types import SimpleNamespace
import routes.gallery_routes as gallery_routes
class _Predicate:
def __init__(self, check):
self._check = check
def __call__(self, row):
return self._check(row)
def __or__(self, other):
return _Predicate(lambda row: self(row) or other(row))
class _Column:
def __init__(self, name):
self.name = name
def __eq__(self, value):
return _Predicate(lambda row: getattr(row, self.name) == value)
class _ModelEndpoint:
base_url = _Column("base_url")
model_type = _Column("model_type")
is_enabled = _Column("is_enabled")
owner = _Column("owner")
class _Query:
def __init__(self, rows):
self._rows = list(rows)
def filter(self, *predicates):
self._rows = [row for row in self._rows if all(pred(row) for pred in predicates)]
return self
def all(self):
return list(self._rows)
class _DB:
def __init__(self, rows):
self._rows = rows
def query(self, model):
assert model is _ModelEndpoint
return _Query(self._rows)
def _ep(base_url, owner=None, *, enabled=True, model_type="chat", models=None):
return SimpleNamespace(
base_url=base_url,
owner=owner,
is_enabled=enabled,
model_type=model_type,
api_key="sk-secret",
cached_models=models or [],
pinned_models=[],
)
def _patch_model(monkeypatch):
monkeypatch.setattr(gallery_routes, "ModelEndpoint", _ModelEndpoint)
def test_mixed_endpoint_with_image_model_is_found(monkeypatch):
_patch_model(monkeypatch)
rows = [_ep("https://gateway.example.com/v1", models=["qwen-image-edit", "llama-3"])]
ep = gallery_routes._first_visible_image_endpoint(_DB(rows), None)
assert ep is not None
assert ep.base_url == "https://gateway.example.com/v1"
def test_mixed_endpoint_without_image_model_is_ignored(monkeypatch):
_patch_model(monkeypatch)
rows = [_ep("https://gateway.example.com/v1", models=["llama-3", "mistral"])]
assert gallery_routes._first_visible_image_endpoint(_DB(rows), None) is None
def test_image_only_endpoint_wins_over_mixed(monkeypatch):
_patch_model(monkeypatch)
rows = [
_ep("https://gateway.example.com/v1", models=["gpt-image-1"]),
_ep("https://images.example.com/v1", model_type="image"),
]
ep = gallery_routes._first_visible_image_endpoint(_DB(rows), None)
assert ep is not None
assert ep.model_type == "image"
def test_unreachable_endpoint_reports_502():
"""The inpaint proxy must distinguish an unreachable endpoint from a provider error.
502 (gateway) is reserved for "the image server could not be reached"; 503 is
reserved for "generation is switched off on the server", which the editor maps
to its own fixed message.
"""
from pathlib import Path
src = (
Path(__file__).resolve().parents[1] / "routes" / "gallery" / "gallery_routes.py"
).read_text()
assert "Image server unreachable" in src
assert "except httpx.TransportError as exc:" in src
assert "provider HTTP" in src
assert 'raise HTTPException(503, f"Image endpoint unreachable' not in src
+39
View File
@@ -0,0 +1,39 @@
"""Source checks for the gallery Settings-tab image-model picker.
The Quick Edit model picker lives in the editor; HARNESS mirrors it into the
gallery Settings tab, fed by GET /api/editor/image-models and sharing the
same localStorage key (`ge-ai-command-model`) so the two stay in sync.
"""
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
GALLERY_JS = (ROOT / "static/js/gallery.js").read_text(encoding="utf-8")
def test_settings_tab_has_image_model_picker():
assert 'id="gallery-image-model"' in GALLERY_JS
assert "/api/editor/image-models" in GALLERY_JS
assert "ge-ai-command-model" in GALLERY_JS
def test_image_model_picker_lives_in_the_settings_container():
settings_idx = GALLERY_JS.index('id="gallery-settings-container"')
picker_idx = GALLERY_JS.index('id="gallery-image-model"')
assert picker_idx > settings_idx
def test_picker_persists_with_the_shared_localstorage_key():
assert "localStorage.setItem(_GE_MODEL_KEY" in GALLERY_JS
assert "localStorage.getItem(_GE_MODEL_KEY)" in GALLERY_JS
def test_cloud_image_models_are_listed_alongside_the_local_ones():
"""Gateway endpoints (OpenRouter) register as llm but carry image models.
The picker must surface those ids too — otherwise the only selectable
edit model is the local Qwen server.
"""
routes = (ROOT / "routes/editor_agent_routes.py").read_text(encoding="utf-8")
assert "from src.image_model_ids import looks_like_image_generation_model" in routes
assert 'image_endpoint = getattr(ep, "model_type", None) == "image"' in routes
assert "if not image_endpoint and not looks_like_image_generation_model(name):" in routes
@@ -0,0 +1,25 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
GALLERY = ROOT / "static/js/gallery.js"
def test_editor_landing_has_an_import_button_next_to_new_canvas():
"""The editor landing must offer a direct "Import" action.
It sits in the same action row as "New canvas..." / "Browse gallery" and,
on file pick, opens the editor on the chosen raster image (``openEditor``
takes an image data URL as its first argument).
"""
src = GALLERY.read_text(encoding="utf-8")
assert 'id="gallery-editor-import"' in src, "landing Import button missing"
# Button order: New canvas → Import → Browse gallery.
new_at = src.index('id="gallery-editor-new"')
import_at = src.index('id="gallery-editor-import"')
pick_at = src.index('id="gallery-editor-pick"')
assert new_at < import_at < pick_at, "Import must sit next to New canvas"
# Wiring opens the editor on the selected file's data URL.
assert "getElementById('gallery-editor-import')?.addEventListener('click'" in src
assert "readAsDataURL(file)" in src
assert "await openEditor(reader.result, null, null, name)" in src
+187
View File
@@ -0,0 +1,187 @@
"""Tests for the seneca single-image op proxies (HARNESS-owned).
POST /api/image/replace-background -> provider /v1/images/replace-background
POST /api/image/outpaint -> provider /v1/images/outpaint (alias /extend)
Both forward a client-supplied JPEG/PNG as multipart with the prompt/params plus
``request_id`` (so GET /api/image/edit-progress lights the progress bar) and
return the provider image as ``{image}``.
"""
import asyncio
import base64
from types import SimpleNamespace
import httpx
import routes.gallery_routes as gallery
_IMG_B64 = base64.b64encode(b"abc-not-a-real-png").decode()
def _route(path):
return next(r for r in gallery.setup_gallery_routes().routes if r.path == path)
class _Req:
def __init__(self, body):
self._body = body
async def json(self):
return self._body
def _fake_db():
class Db:
def close(self):
pass
def query(self, *args, **kwargs):
return self
def filter(self, *args, **kwargs):
return self
def first(self):
return None
return Db()
def _setup(monkeypatch, respond):
endpoint = SimpleNamespace(base_url="http://100.69.120.65:8188/v1", api_key="k")
monkeypatch.setattr(gallery, "SessionLocal", lambda: _fake_db())
monkeypatch.setattr(gallery, "require_privilege", lambda *a: "alice")
monkeypatch.setattr(gallery, "_first_visible_image_endpoint", lambda *a: endpoint)
seen = {}
def _respond(request):
seen["url"] = str(request.url)
seen["body"] = bytes(request.content)
return respond(request)
client = httpx.AsyncClient
monkeypatch.setattr(
httpx, "AsyncClient",
lambda **k: client(transport=httpx.MockTransport(_respond), **k),
)
return seen
def _ok(_req):
return httpx.Response(200, json={"data": [{"b64_json": "AAAA"}]})
def test_replace_background_forwards_multipart_and_returns_image(monkeypatch):
seen = _setup(monkeypatch, _ok)
out = asyncio.run(_route("/api/image/replace-background").endpoint(_Req({
"image": _IMG_B64, "prompt": "put her on a beach", "steps": 8,
"harmonise": True, "shadow": False, "request_id": "rb-123",
})))
assert out["image"] == "AAAA"
assert seen["url"].endswith("/images/replace-background")
body = seen["body"]
assert b"put her on a beach" in body
assert b"rb-123" in body # request_id forwarded for progress
assert b"harmonise" in body and b"shadow" in body
assert b"8" in body # steps
def test_outpaint_forwards_ratio_size_context_and_returns_image(monkeypatch):
seen = _setup(monkeypatch, _ok)
out = asyncio.run(_route("/api/image/outpaint").endpoint(_Req({
"image": _IMG_B64, "prompt": "extend the sky", "ratio": "16:9",
"size": "1920x1080", "steps": 4, "context": "calm blue sky",
"request_id": "op-9",
})))
assert out["image"] == "AAAA"
assert seen["url"].endswith("/images/outpaint")
body = seen["body"]
for token in (b"16:9", b"1920x1080", b"calm blue sky", b"op-9"):
assert token in body, token
def test_outpaint_extend_alias_uses_the_same_provider_route(monkeypatch):
seen = _setup(monkeypatch, _ok)
out = asyncio.run(_route("/api/image/extend").endpoint(_Req({
"image": _IMG_B64, "prompt": "extend", "ratio": "9:16",
})))
assert out["image"] == "AAAA"
assert seen["url"].endswith("/images/outpaint")
def test_outpaint_reads_openai_style_url_result(monkeypatch):
def _respond(_req):
return httpx.Response(200, json={"data": [{"url": "http://100.69.120.65:8188/v1/images/x.png"}]})
_setup(monkeypatch, _respond)
monkeypatch.setattr(gallery, "_fetch_result_image_b64", lambda url: _async_return("ZZZZ"))
out = asyncio.run(_route("/api/image/outpaint").endpoint(_Req({"image": _IMG_B64, "prompt": "x"})))
assert out["image"] == "ZZZZ"
def test_replace_background_502_when_provider_returns_no_image(monkeypatch):
_setup(monkeypatch, lambda _req: httpx.Response(200, json={"data": []}))
try:
asyncio.run(_route("/api/image/replace-background").endpoint(_Req({"image": _IMG_B64})))
except Exception as exc:
assert getattr(exc, "status_code", None) == 502
else:
raise AssertionError("expected a 502 when the provider returns no image")
async def _async_return(value):
return value
def _capture_forward(monkeypatch):
"""Replace _forward_seneca_image_op with a fake that records kwargs."""
captured = {}
async def _fake_forward(base, api_key, provider_path, image_bytes, fields, *, label, timeout=300.0):
captured["label"] = label
captured["provider_path"] = provider_path
captured["timeout"] = timeout
return {"image": "AAAA"}
monkeypatch.setattr(gallery, "_forward_seneca_image_op", _fake_forward)
monkeypatch.setattr(gallery, "SessionLocal", lambda: _fake_db())
monkeypatch.setattr(gallery, "require_privilege", lambda *a: "alice")
monkeypatch.setattr(
gallery, "_first_visible_image_endpoint",
lambda *a: SimpleNamespace(base_url="http://100.69.120.65:8188/v1", api_key="k"),
)
return captured
def test_outpaint_proxy_uses_a_long_timeout(monkeypatch):
captured = _capture_forward(monkeypatch)
out = asyncio.run(_route("/api/image/outpaint").endpoint(_Req({"image": _IMG_B64, "prompt": "extend to 9:16"})))
assert out["image"] == "AAAA"
assert captured["provider_path"] == "/images/outpaint"
assert captured["timeout"] >= 900, captured
def test_outpaint_extend_alias_also_uses_the_long_timeout(monkeypatch):
captured = _capture_forward(monkeypatch)
asyncio.run(_route("/api/image/extend").endpoint(_Req({"image": _IMG_B64, "prompt": "extend"})))
assert captured["timeout"] >= 900, captured
def test_replace_background_keeps_the_short_default_timeout(monkeypatch):
captured = _capture_forward(monkeypatch)
asyncio.run(_route("/api/image/replace-background").endpoint(_Req({"image": _IMG_B64, "prompt": "beach"})))
assert captured["timeout"] == 300.0, captured
def test_outpaint_timeout_env_override(monkeypatch):
monkeypatch.setenv("ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT", "720")
assert gallery._outpaint_timeout() == 720.0
monkeypatch.delenv("ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT", raising=False)
assert gallery._outpaint_timeout() == 900.0
def test_graceful_shutdown_window_covers_a_long_outpaint():
from pathlib import Path
src = (Path(__file__).resolve().parent.parent / "app.py").read_text()
assert '"ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT", "900"' in src
+4 -2
View File
@@ -120,8 +120,9 @@ def test_disabled_generation_is_not_restored():
@pytest.mark.asyncio
@pytest.mark.parametrize('incognito', [False, True])
@pytest.mark.parametrize('exit_code', [None, 0])
async def test_generation_dispatch_uses_owner_aware_backend(monkeypatch, exit_code):
async def test_generation_dispatch_uses_owner_aware_backend(monkeypatch, exit_code, incognito):
from types import SimpleNamespace
from src import ai_interaction, tool_execution
from src.agent_runtime.journal import ActionJournal, bind_journal
@@ -145,6 +146,7 @@ async def test_generation_dispatch_uses_owner_aware_backend(monkeypatch, exit_co
_, denied = await execute(block, owner='pewds', session_id='fixture',
disabled_tools={'generate_image'}, security_context=tool_execution.NO_TOOL_SECURITY_CONTEXT)
_, result = await execute(block, owner='pewds', session_id='fixture',
client_runtime_context={'web_ui_state': {'incognito': incognito}},
security_context=tool_execution.NO_TOOL_SECURITY_CONTEXT)
assert denied['exit_code'] != 0
assert journal.actions[0].execution_id is None
@@ -152,7 +154,7 @@ async def test_generation_dispatch_uses_owner_aware_backend(monkeypatch, exit_co
assert journal.actions[1].execution_id is not None
assert journal.actions[1].outcome['authoritative'] is (exit_code == 0)
assert result['image_id'] == 'test'
assert calls == [(block.content, {'owner': 'pewds', 'session_id': 'fixture'})]
assert calls == [(block.content, {'owner': 'pewds', 'session_id': 'fixture', **({'incognito': True} if incognito else {})})]
@pytest.mark.asyncio
+54
View File
@@ -0,0 +1,54 @@
"""The heavy image-edit CPU work must run off the event loop, and the app must
expose in-flight heavy-op counts + a graceful shutdown so a deploy restart does
not drop a 60-150 s edit."""
import base64
import io
from pathlib import Path
from PIL import Image
from routes.gallery import gallery_routes as gr
ROOT = Path(__file__).resolve().parents[1]
def _png_b64(w, h, color=(120, 160, 210)):
buf = io.BytesIO()
Image.new("RGBA", (w, h), color).save(buf, format="PNG")
return base64.b64encode(buf.getvalue()).decode("ascii")
def test_prepare_local_edit_caps_large_input():
src, mask, mw, mh, orig = gr._prepare_local_edit(_png_b64(4000, 3000), None, 768)
assert orig == (4000, 3000)
assert max(mw, mh) <= 768 and mw % 16 == 0 and mh % 16 == 0
assert mask # a full-white mask is synthesised when none is supplied
assert Image.open(io.BytesIO(src)).size == (mw, mh)
def test_prepare_local_edit_leaves_small_input_unchanged():
_src, _mask, mw, mh, orig = gr._prepare_local_edit(_png_b64(96, 96), _png_b64(96, 96, (255, 255, 255)), 768)
assert orig == (96, 96) and (mw, mh) == (96, 96)
def test_restore_local_b64_resizes_back_to_original():
raw = _png_b64(768, 576)
out = gr._restore_local_b64(raw, (4000, 3000), (768, 576))
assert Image.open(io.BytesIO(base64.b64decode(out))).size == (4000, 3000)
# a same-size result is returned untouched
assert gr._restore_local_b64(raw, (768, 576), (768, 576)) == raw
def test_inpaint_offloads_cpu_work_to_a_thread():
src = (ROOT / "routes/gallery/gallery_routes.py").read_text()
assert "await asyncio.to_thread(\n _prepare_local_edit" in src
assert "asyncio.to_thread(\n _restore_local_b64" in src
def test_app_exposes_busy_count_and_graceful_shutdown():
src = (ROOT / "app.py").read_text()
assert "busy_image_ops" in src
assert '@app.get("/api/busy")' in src
assert "_HEAVY_IMAGE_OP_PREFIXES" in src
assert "timeout_graceful_shutdown" in src
assert "activity.begin_image_op" in src and "activity.end_image_op" in src
+32
View File
@@ -33,6 +33,38 @@ def test_native_generation_respects_admin_disable(monkeypatch):
assert 'disabled' in result['error']
@pytest.mark.parametrize('editing', [False, True])
def test_incognito_images_never_write_gallery_or_files(image_transport, editing):
configure, source, requests, rows = image_transport
configure('https://openrouter.ai/api/v1')
if editing:
result = asyncio.run(ai_interaction.do_edit_image(
'Private edit', str(source), model_spec='openai/gpt-5-image',
owner='alice', session_id='private-chat', incognito=True))
else:
result = asyncio.run(ai_interaction.do_generate_image(
'Private image\nopenai/gpt-5-image', owner='alice',
session_id='private-chat', incognito=True))
assert 'error' not in result
assert result['image_url'].startswith('data:image/png;base64,')
assert base64.b64decode(result['image_url'].split(',', 1)[1]) == source.read_bytes()
assert not result['image_id']
assert not rows
assert not (source.parent / 'generated').exists()
def test_incognito_url_image_is_not_persisted(image_transport, monkeypatch):
configure, source, requests, rows = image_transport
configure('https://api.openai.com/v1', body={'data': [{'url': 'https://example.com/output.png'}]})
monkeypatch.setattr(httpx, 'get', lambda *args, **kwargs: httpx.Response(200, content=source.read_bytes()))
monkeypatch.setattr('src.url_safety.check_outbound_url', lambda *args, **kwargs: (True, ''))
result = asyncio.run(ai_interaction.do_generate_image(
'Private image\nopenai/gpt-5-image', owner='alice', incognito=True))
assert result['image_url'].startswith('data:image/png;base64,')
assert not rows
assert not (source.parent / 'generated').exists()
@pytest.fixture
def image_transport(monkeypatch, tmp_path):
rows = []
+503
View File
@@ -0,0 +1,503 @@
"""Scoped ``photo_editor`` capability family: routing, tools, and the
Quick Edit agent endpoint.
The family is context-only: it is offered to the model ONLY when an editor
session / edit-this-photo request explicitly selects it, never inferred from
message text and never part of the general tool list.
"""
import inspect
import pytest
from src.turn_contract import (
FAMILY_TOOLS,
requested_capabilities,
resolve_full_inventory_contract,
resolve_turn_contract,
)
from src.tool_schemas import FUNCTION_TOOL_SCHEMAS
from src.tool_policy import ToolPolicy
from src.tool_types import TOOL_TAGS
from src.agent_tools import TOOL_HANDLERS
from src.tool_index import BUILTIN_TOOL_DESCRIPTIONS
from src.tool_capabilities import KNOWN_CAPABILITY_TOOLS
from src.tool_security import PLAN_MODE_READONLY_TOOLS, _PLAN_MODE_KNOWN_MUTATORS
PHOTO_TOOLS = frozenset(
{"plan_edit", "run_editor_command", "get_editor_screenshot", "ai_edit_image"}
)
FAMILY = FAMILY_TOOLS["photo_editor"]
CORE_NAMES = frozenset({"bash", "python", "web_search", "web_fetch", "write_file"})
def _schema_names():
return {s["function"]["name"] for s in FUNCTION_TOOL_SCHEMAS}
def test_family_contains_scoped_tools_only():
assert FAMILY == PHOTO_TOOLS | {"generate_image"}
def test_contract_offers_only_editor_tools_and_ask_user():
contract = resolve_turn_contract(
capabilities={"photo_editor"},
schemas=FUNCTION_TOOL_SCHEMAS,
policy=ToolPolicy(),
required_capabilities={"photo_editor"},
)
assert contract.offered == FAMILY | {"ask_user", "update_plan"}
assert not (CORE_NAMES & contract.offered)
def test_photo_editor_is_never_inferred_from_message_text():
for message in (
"make it black and white",
"generate an image of a cat",
"remove the person on the left",
"add a full moon in the sky",
"blur the background behind the person",
):
caps = requested_capabilities(message)
assert "photo_editor" not in caps, message
def test_generate_image_keeps_its_own_family_without_leaking_photo_editor():
caps = requested_capabilities("Create a note and generate an image of a cat")
assert "image_generation" in caps
assert "photo_editor" not in caps
def test_editor_tools_are_registered_everywhere():
schema_names = _schema_names()
for name in PHOTO_TOOLS:
assert name in TOOL_TAGS, name
assert name in TOOL_HANDLERS, name
assert name in BUILTIN_TOOL_DESCRIPTIONS, name
assert name in KNOWN_CAPABILITY_TOOLS, name
assert name in schema_names, name
assert "generate_image" in FAMILY
def test_editor_tools_plan_mode_partition():
assert "get_editor_screenshot" in PLAN_MODE_READONLY_TOOLS
for name in ("plan_edit", "run_editor_command", "ai_edit_image"):
assert name in _PLAN_MODE_KNOWN_MUTATORS, name
def test_full_inventory_contains_the_editor_tools():
preview = resolve_full_inventory_contract(
schemas=FUNCTION_TOOL_SCHEMAS, policy=ToolPolicy()
)
assert FAMILY <= set(preview.offered)
def test_scope_preview_contract_keeps_only_the_editor_family():
from dataclasses import replace
from src.clean_agent_preview import scope_preview_contract
preview = resolve_full_inventory_contract(
schemas=FUNCTION_TOOL_SCHEMAS, policy=ToolPolicy()
)
routed = resolve_turn_contract(
capabilities={"photo_editor"},
schemas=FUNCTION_TOOL_SCHEMAS,
policy=ToolPolicy(),
required_capabilities={"photo_editor"},
)
scoped = scope_preview_contract(
replace(preview, selection_mode="full_compact_experiment"),
routed,
frozenset({"photo_editor"}),
extra_tools=frozenset({"ask_user"}),
)
assert FAMILY <= set(scoped.offered)
assert "ask_user" in scoped.offered
assert not (CORE_NAMES & set(scoped.offered))
def test_editor_agent_endpoint_reports_scoped_tools():
from fastapi import FastAPI
from fastapi.testclient import TestClient
import routes.editor_agent_routes as eaar
eaar.set_editor_planner(lambda messages, owner: {"reply": "ok", "steps": [], "done": True})
try:
app = FastAPI()
app.include_router(eaar.setup_editor_agent_routes())
client = TestClient(app)
resp = client.post(
"/api/editor/agent",
json={"prompt": "hello there", "session": "s1"},
)
assert resp.status_code == 200
data = resp.json()
assert data["ok"] is True
assert PHOTO_TOOLS <= set(data["tools_offered"])
assert "generate_image" in data["tools_offered"]
assert not (CORE_NAMES & set(data["tools_offered"]))
assert data["trace"][0]["type"] == "session"
finally:
eaar.set_editor_planner(None)
async def _route_contract(monkeypatch, message, *, attachment_meta=()):
from routes import chat_routes
from src import tool_security
from tests.test_foreground_model_routing import _RouteRequest, _chat_stream_endpoint
with monkeypatch.context() as route_patch:
endpoint = _chat_stream_endpoint(
route_patch, "agent", {},
session_model="odysseus-qwen3.5-tools-pre-heretic",
attachment_meta=attachment_meta,
)
route_patch.setattr(
chat_routes, "coerce_message_and_session",
lambda *args, **kwargs: (message, "session-1"),
)
route_patch.setattr(
tool_security, "owner_is_admin_or_single_user", lambda owner: True,
)
observed = []
async def capture_agent(*args, **kwargs):
observed.append(kwargs["turn_contract"])
yield 'data: {"delta":"ok"}\n\n'
yield "data: [DONE]\n\n"
route_patch.setattr(chat_routes, "stream_agent_loop", capture_agent)
request = _RouteRequest("agent")
request._form.update({
"message": message, "compare_mode": "false",
"allow_web_search": "false", "allow_bash": "false",
})
response = await endpoint(request)
async for _ in response.body_iterator:
pass
assert len(observed) == 1
return observed[0]
@pytest.mark.asyncio
async def test_normal_chat_turn_offers_no_editor_tools(monkeypatch):
for message in ("what is the capital of France", "write a haiku"):
contract = await _route_contract(monkeypatch, message)
assert "photo_editor" not in contract.capabilities, message
assert not (PHOTO_TOOLS & set(contract.offered)), message
@pytest.mark.asyncio
async def test_attached_photo_edit_selects_photo_editor_family(monkeypatch):
contract = await _route_contract(
monkeypatch,
"make this black and white",
attachment_meta=[{"id": "att1", "mime": "image/png", "name": "x.png"}],
)
assert contract.capabilities == {"photo_editor"}
assert set(contract.offered) == FAMILY | {"ask_user"}
assert not (CORE_NAMES & set(contract.offered))
def test_chat_context_guides_open_or_absent_editor():
from pathlib import Path
src = (Path(__file__).resolve().parent.parent / "routes" / "chat_routes.py").read_text(
encoding="utf-8"
)
assert "The image editor is open (session " in src
assert "no image editor is open" in src
assert "the attached image in the editor (Quick Edit)" in src
def test_generative_intent_classification():
from routes.editor_agent_routes import _generative_kind
assert _generative_kind("remove the person on the left") == "remove"
assert _generative_kind("erase the sign in the background") == "remove"
assert _generative_kind("add a full moon in the sky") == "add"
assert _generative_kind("put a bird on the fence") == "add"
assert _generative_kind("replace the sky with a sunset") == "replace"
assert _generative_kind("change the background to night") == "replace"
# Non-generative edits must stay deterministic.
assert _generative_kind("make it black and white") is None
assert _generative_kind("make it brighter") is None
assert _generative_kind("add a vignette") is None
assert _generative_kind("remove the background") is None
assert _generative_kind("sharpen the image") is None
def test_generative_step_uses_ai_content_tools():
from routes.editor_agent_routes import _generative_step
assert _generative_step("remove", "remove the person")["op"] == "ai_edit_image"
assert _generative_step("replace", "replace the sky")["op"] == "ai_edit_image"
# Adding content to an existing photo is an instruction edit, not generate_image.
assert _generative_step("add", "add a moon")["op"] == "ai_edit_image"
assert _generative_step("add", "add a moon")["params"]["mode"] == "instruction"
# generate_image only for a brand-new image.
assert _generative_step("add", "create a new image of a moon")["op"] == "generate_image"
def test_generative_mode_instruction_vs_masked():
from routes.editor_agent_routes import _generative_mode
# Masked only for an explicit regional removal of a clearly identified object.
assert _generative_mode("remove", "remove the person on the left") == "masked"
# Diffuse cleanup, replacement, season/style and additions use instruction mode.
assert _generative_mode("remove", "remove the distractions") == "instruction"
assert _generative_mode("replace", "make the background nicer") == "instruction"
assert _generative_mode("add", "add a full moon in the sky") == "instruction"
def test_ensure_generative_replaces_deterministic_approximations():
from routes.editor_agent_routes import _ensure_generative
bad = [{"op": "selection.rectangle", "params": {}}, {"op": "adjustments.apply", "params": {}}]
removal = _ensure_generative("remove the person on the left", list(bad))
# A mask selection step now precedes the masked AI edit.
assert removal[-1]["op"] == "ai_edit_image"
assert removal[-1]["params"]["mode"] == "masked"
assert all(s["op"].startswith("selection.") for s in removal[:-1])
# Adding an object to an existing photo is an instruction edit (no selection).
addition = _ensure_generative("add a full moon in the sky", list(bad))
assert addition[-1]["op"] == "ai_edit_image"
assert addition[-1]["params"]["mode"] == "instruction"
# Non-generative plans are left untouched.
assert _ensure_generative("make it black and white", list(bad)) == bad
def test_fallback_plan_routes_generative_intents():
import routes.editor_agent_routes as eaar
removal = eaar._fallback_plan("remove the person on the left")
assert removal["steps"][-1]["op"] == "ai_edit_image"
assert removal["steps"][-1]["params"]["mode"] == "masked"
addition = eaar._fallback_plan("add a full moon in the sky")
assert addition["steps"][-1]["op"] == "ai_edit_image"
assert addition["steps"][-1]["params"]["mode"] == "instruction"
plain = eaar._fallback_plan("make it black and white")
assert plain["steps"][0]["op"] == "adjustments.apply"
def test_provider_message_is_actionable():
from src.ai_interaction import IMAGE_PROVIDER_REQUIRED_MESSAGE
assert "image generation provider" in IMAGE_PROVIDER_REQUIRED_MESSAGE
assert "Settings > Models" in IMAGE_PROVIDER_REQUIRED_MESSAGE
def test_image_provider_respects_disabled_setting(monkeypatch):
import src.settings as settings
monkeypatch.setattr(settings, "load_settings", lambda: {"image_gen_enabled": False})
from src.ai_interaction import image_editing_provider_available
assert image_editing_provider_available(None) is False
def _agent_client():
from fastapi import FastAPI
from fastapi.testclient import TestClient
import routes.editor_agent_routes as eaar
app = FastAPI()
app.include_router(eaar.setup_editor_agent_routes())
return eaar, TestClient(app)
@pytest.mark.parametrize(
"prompt",
["remove the person on the left", "add a full moon in the sky"],
)
def test_editor_agent_blocks_generative_without_provider(monkeypatch, prompt):
"""No image provider -> zero canvas changes + a plain provider message."""
eaar, client = _agent_client()
# Planner proposes an unrelated deterministic edit, as the buggy run did.
eaar.set_editor_planner(
lambda messages, owner: {"reply": "done", "steps": [
{"op": "selection.rectangle", "params": {}},
{"op": "adjustments.apply", "params": {}},
]}
)
monkeypatch.setattr(eaar, "_image_provider_available", lambda owner: False)
try:
resp = client.post(
"/api/editor/agent", json={"prompt": prompt, "session": "s1"}
)
assert resp.status_code == 200
data = resp.json()
assert data["needs_provider"] is True
assert data["blocked"] is True
assert "image generation provider" in data["message"]
assert data["image_provider"] is False
assert data["results"] == []
# Nothing was executed through the bridge (zero canvas changes).
assert not any(t.get("type") == "step" for t in data["trace"])
assert any(t.get("reason") == "image_provider_unavailable" for t in data["trace"])
# The proposed deterministic plan was replaced by the correct AI op.
ops = [s["op"] for s in data["steps"]]
assert ops[-1] in ("ai_edit_image", "generate_image")
finally:
eaar.set_editor_planner(None)
def test_editor_agent_prefers_ai_op_over_deterministic_when_provider_present(monkeypatch):
eaar, client = _agent_client()
eaar.set_editor_planner(
lambda messages, owner: {"reply": "done", "steps": [
{"op": "selection.rectangle", "params": {}},
{"op": "adjustments.apply", "params": {}},
]}
)
monkeypatch.setattr(eaar, "_image_provider_available", lambda owner: True)
async def _instant_wait_for(awaitable, *args, **kwargs):
# The plan now starts with a mask-selection bridge step; resolve it
# immediately instead of waiting for a (non-existent) editor subscriber.
try:
awaitable.close()
except Exception:
pass
return {"ok": True, "result": {"width": 100, "height": 100}}
monkeypatch.setattr(eaar.asyncio, "wait_for", _instant_wait_for)
calls = []
class _FakeAiEdit:
async def execute(self, content, ctx):
calls.append(content)
return "ai_edit_image: fake", {"output": "ok", "exit_code": 0}
class _FakeGen:
async def execute(self, content, ctx):
calls.append(content)
return "generate_image: fake", {"output": "ok", "exit_code": 0}
import src.agent_tools.editor_tools as et
monkeypatch.setattr(et, "AiEditImageTool", _FakeAiEdit)
monkeypatch.setattr(et, "GenerateImageLayerTool", _FakeGen)
try:
resp = client.post(
"/api/editor/agent",
json={"prompt": "remove the person on the left", "session": "s1"},
)
assert resp.status_code == 200
data = resp.json()
assert data["ok"] is True
ops = [s["op"] for s in data["steps"]]
assert ops[-1] == "ai_edit_image"
assert any(r.get("op") == "ai_edit_image" for r in data["results"])
assert calls, "the AI edit tool should have been invoked"
assert "adjustments.apply" not in ops
finally:
eaar.set_editor_planner(None)
def test_editor_prompts_advertise_canonical_shapes():
import routes.editor_agent_routes as eaar
canonical = eaar._CANONICAL_GUIDANCE
assert "brightness-contrast" in canonical and "render-clouds" in canonical
assert canonical in eaar._SYSTEM_PROMPT
assert canonical in eaar._CHECK_SYSTEM_PROMPT
assert "_CANONICAL_GUIDANCE" in inspect.getsource(eaar.setup_editor_agent_routes)
def test_run_editor_command_schema_advertises_canonical_shapes():
from src.tool_schemas import FUNCTION_TOOL_SCHEMAS
schema = next(s for s in FUNCTION_TOOL_SCHEMAS
if s["function"]["name"] == "run_editor_command")
desc = schema["function"]["description"]
assert "brightness-contrast" in desc and "render-clouds" in desc
assert "vignette" in desc # names the banned invented ids
def test_photo_editor_chat_without_canvas_asks_to_open_editor():
from src import clean_agent_preview
source = inspect.getsource(clean_agent_preview.stream_preview)
assert "not open on the editor canvas" in source
assert "'photo_editor' in (getattr(turn_contract, 'active_capabilities'" in source
def test_chat_photo_edit_without_open_editor_short_circuits(monkeypatch):
"""A photo-edit chat turn with no open editor must not run the tool loop.
Regression: offering the editor tools with no canvas made the model loop
on plan_edit/run_editor_command until the round cap and end with an empty
answer. The runtime now answers deterministically and asks the user to
open the image in the editor.
"""
import asyncio
import json as _json
from dataclasses import replace
import src.clean_agent_preview as module
contract = replace(
resolve_full_inventory_contract(schemas=FUNCTION_TOOL_SCHEMAS, policy=ToolPolicy()),
active_capabilities=frozenset({"photo_editor"}),
)
provider_calls = []
class _Client:
def __init__(self, **kwargs):
pass
async def __aenter__(self):
return self
async def __aexit__(self, *args):
pass
def stream(self, *args, **kwargs):
provider_calls.append(kwargs.get("json"))
raise AssertionError("provider must not be called when no editor is open")
monkeypatch.setattr(module.httpx, "AsyncClient", _Client)
async def _run():
return [
chunk
async for chunk in module.stream_preview(
endpoint_url="http://test",
model="test",
headers={},
messages=[{"role": "user", "content": "make this black and white"}],
turn_contract=contract,
session_id="test",
owner="test",
disabled_tools=set(),
tool_policy=ToolPolicy(),
client_runtime_context={"web_ui_state": {"photo_editor_open": False}},
)
]
chunks = asyncio.run(_run())
events = [
_json.loads(chunk[6:])
for chunk in chunks
if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]")
]
assert provider_calls == []
contract_event = next(e for e in events if e.get("type") == "turn_contract")
assert "photo_editor" in contract_event["active_capabilities"]
final = next(e for e in events if e.get("type") == "final_response")
lowered = final["content"].lower()
assert "editor" in lowered and "open" in lowered
# One-click affordance contract for the chat UI.
assert final["photo_editor_action"] == {
"kind": "open_and_apply",
"prompt": "make this black and white",
}
assert not any(e.get("type") == "tool_start" for e in events)
if __name__ == "__main__": # pragma: no cover
raise SystemExit(pytest.main([__file__, "-q"]))
+642
View File
@@ -0,0 +1,642 @@
"""Unit tests for the shared photo-editor style guide and its server wiring.
The guide (``src/editor_style_guide.py``) is the single source of truth for
intent -> ops routing, calibrated strengths and the low-key / temperature /
portrait / ambiguity rules. It feeds both the server planner and (via
``GET /api/editor/style-guide``) the browser fallback.
"""
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
import routes.editor_agent_routes as eaar
from src import editor_style_guide as guide
@pytest.fixture(autouse=True)
def _reset_planner():
eaar.set_editor_planner(None)
yield
eaar.set_editor_planner(None)
@pytest.fixture()
def client():
app = FastAPI()
app.include_router(eaar.setup_editor_agent_routes())
return TestClient(app)
def _ops(steps):
return [s["op"] for s in steps]
def test_new_ops_are_in_the_server_op_catalog():
# If a new retouch/selection op is missing from OP_CATALOG, _parse_plan
# filters a valid plan step out and the edit silently disappears.
for name in guide.NEW_OP_NAMES:
assert name in eaar.OP_CATALOG
for name in ("photo.autoTone", "retouch.skin", "retouch.teeth",
"selection.face", "selection.background"):
assert name in eaar._OP_SET
def test_portrait_skin_intent_is_a_multi_step_plan():
plan = guide.fallback_plan("can you fix the skin a bit, look my best")
assert plan is not None
# No leading whole-image auto tone: it washes out the face. Go straight to a
# gentle face retouch, then clear the selection.
assert _ops(plan["steps"]) == [
"selection.face", "retouch.skin", "selection.deselect",
]
skin = plan["steps"][1]["params"]
assert 0 < skin["smooth"] <= 14
assert 0 < skin["opacity"] <= 0.45
# No global lift anywhere in the portrait chain (client never brightens skin).
assert not any("autoTone" in op or "brightness" in op or "exposure" in op
for op in _ops(plan["steps"]))
def test_portrait_intent_does_not_lead_with_auto_tone():
for prompt in ("make me look better", "fix the skin", "fix my teeth",
"smooth my complexion", "portrait touch up"):
plan = guide.fallback_plan(prompt)
assert plan is not None, prompt
assert "photo.autoTone" not in _ops(plan["steps"]), prompt
def test_teeth_intent_selects_face_then_whitens():
plan = guide.fallback_plan("fix my teeth")
assert _ops(plan["steps"]) == ["selection.face", "retouch.teeth"]
assert plan["steps"][1]["params"]["desat"] > 0
def test_background_intent_is_selection_then_masked_blur():
plan = guide.fallback_plan("blur the background / make the background nicer")
ops = _ops(plan["steps"])
assert ops == ["selection.background", "filters.applyMasked"]
assert plan["steps"][1]["params"]["id"] == "blur-gaussian"
def test_temperature_sign_is_cool_negative_warm_positive():
cool = guide.fallback_plan("make it moodier")
assert cool["steps"][0]["params"] == {
"type": "white-balance", "params": {"temperature": -25}}
warm = guide.fallback_plan("make it warmer, golden summer feel")
assert warm["steps"][0]["params"]["params"]["temperature"] == 25
def test_lighting_intent_uses_auto_tone_then_single_shadow_lift():
plan = guide.fallback_plan("fix the lighting, it's too dark")
assert _ops(plan["steps"]) == ["photo.autoTone", "adjustments.apply"]
inner = plan["steps"][1]["params"]
assert inner["type"] == "shadows-highlights"
assert inner["params"]["highlights"] <= 0
# The sanctioned low-key pair, never a stack of unspecified lifts.
assert len(plan["steps"]) == 2
def test_ambiguous_prompt_asks_the_user_with_concrete_options():
plan = guide.fallback_plan("clean this up")
assert plan["steps"] == []
ask = plan["ask_user"]
assert ask["question"]
assert 2 <= len(ask["options"]) <= 3
assert all(o.get("label") for o in ask["options"])
assert guide.is_ambiguous("clean this up") is True
# A concrete defect is not ambiguous.
assert guide.is_ambiguous("fix the lighting") is False
def test_collapse_global_lifts_replaces_a_stack():
# With an auto tone present, extra brightness/exposure lifts are dropped
# (the auto tone is the single sanctioned lift).
with_auto = [
{"op": "photo.autoTone", "params": {}},
{"op": "adjustments.apply",
"params": {"type": "brightness-contrast", "params": {"brightness": 30}}},
{"op": "adjustments.apply",
"params": {"type": "exposure", "params": {"exposure": 20}}},
]
out = guide.collapse_global_lifts(with_auto)
assert len(out) == 1
assert out[0]["op"] == "photo.autoTone"
# Without an auto tone, a stack collapses to one safe shadows-highlights.
no_auto = [
{"op": "adjustments.apply",
"params": {"type": "brightness-contrast", "params": {"brightness": 30}}},
{"op": "adjustments.apply",
"params": {"type": "exposure", "params": {"exposure": 20}}},
]
out2 = guide.collapse_global_lifts(no_auto)
assert len(out2) == 1
assert out2[0]["params"]["type"] == "shadows-highlights"
assert out2[0]["params"]["params"]["highlights"] <= 0
# The sanctioned low-key pair (auto tone + one shadows-highlights) is kept.
pair = [
{"op": "photo.autoTone", "params": {}},
{"op": "adjustments.apply",
"params": {"type": "shadows-highlights",
"params": {"shadows": 30, "highlights": -15}}},
]
assert guide.collapse_global_lifts(pair) == pair
def test_collapse_global_lifts_clamps_a_single_brightness_lift():
steps = [{"op": "adjustments.apply",
"params": {"type": "brightness-contrast", "params": {"brightness": 80}}}]
out = guide.collapse_global_lifts(steps)
assert out[0]["params"]["params"]["brightness"] <= 25
def test_llm_plan_parse_collapses_lifts_and_keeps_new_ops():
raw = (
'{"reply": "x", "steps": ['
'{"op": "retouch.skin", "params": {"smooth": 20}},'
'{"op": "photo.autoTone", "params": {}},'
'{"op": "adjustments.apply", "params": {"type": "exposure", "params": {"exposure": 25}}}'
']}'
)
parsed = eaar._parse_plan(raw)
ops = _ops(parsed["steps"])
assert "retouch.skin" in ops
# Every global lift that trails retouch.skin is dropped (the client guard
# "never brighten skin"), so nothing but the retouch step remains.
assert ops == ["retouch.skin"]
def test_editor_agent_returns_ask_user_for_ambiguous_prompt(client):
eaar.set_editor_planner(lambda messages, owner: '{"reply": "ok", "steps": []}')
resp = client.post("/api/editor/agent", json={"prompt": "clean this up"})
assert resp.status_code == 200
body = resp.json()
assert body.get("ask_user_pending") is True
assert body["steps"] == []
assert 2 <= len(body["ask_user"]["options"]) <= 3
def test_editor_plan_returns_ask_user_for_ambiguous_prompt(client):
eaar.set_editor_planner(lambda messages, owner: None)
resp = client.post("/api/editor/plan", json={"prompt": "clean this up"})
assert resp.status_code == 200
body = resp.json()
assert body["steps"] == []
assert body.get("ask_user")
def test_style_guide_endpoint_serves_the_shared_source(client):
resp = client.get("/api/editor/style-guide")
assert resp.status_code == 200
data = resp.json()
assert data["version"] == guide.STYLE_GUIDE_VERSION
assert {o["op"] for o in data["new_ops"]} == set(guide.NEW_OP_NAMES)
assert data["intents"]
assert "temperature" in data["rules"]["temperature_sign"].lower() or \
"negative" in data["rules"]["temperature_sign"]
# The prompt guidance states the sign rule explicitly.
assert "NEGATIVE" in data["guidance"] and "POSITIVE" in data["guidance"]
def test_prompts_embed_the_style_guidance():
assert guide.guidance() in eaar._SYSTEM_PROMPT
assert guide.guidance() in eaar._CHECK_SYSTEM_PROMPT
@pytest.mark.parametrize("prompt", [
"make it pretty", "make it pop", "professional", "film look",
])
def test_generic_look_intent_is_a_decisive_multi_step_plan(prompt):
plan = guide.fallback_plan(prompt)
assert plan is not None
ops = _ops(plan["steps"])
assert ops[0] == "photo.autoTone"
assert len(ops) >= 3
# A visible vibrance lift and a local-contrast filter are present.
assert any(s["op"] == "adjustments.apply" and s["params"].get("type") == "vibrance"
for s in plan["steps"])
assert any(s["op"] == "filters.apply" and s["params"].get("id") == "clarity"
for s in plan["steps"])
def test_restore_intent_never_large_exposure_lift():
plan = guide.fallback_plan("restore this old photo")
assert plan is not None
ops = _ops(plan["steps"])
assert ops[0] == "photo.autoTone"
assert "filters.apply" in ops # sharpen
# Colour recovery present.
assert any(s["op"] == "adjustments.apply" and s["params"].get("type") == "white-balance"
for s in plan["steps"])
# No exposure lift at all, and the contrast lift keeps brightness neutral.
assert not any(s["params"].get("type") == "exposure" for s in plan["steps"])
for s in plan["steps"]:
if s["params"].get("type") == "brightness-contrast":
assert s["params"]["params"].get("brightness", 0) == 0
def test_strengthen_promotes_a_timid_plan_to_the_recipe():
timid = [{"op": "adjustments.apply",
"params": {"type": "brightness-contrast", "params": {"brightness": 8}}}]
promoted = guide.strengthen("make it pop", timid)
assert promoted != timid
assert _ops(promoted)[0] == "photo.autoTone"
assert len(promoted) >= 3
# A prompt with no recipe leaves the plan untouched.
assert guide.strengthen("nonsense prompt zzz", timid) == timid
def test_enforce_guards_removes_trailing_lift_after_skin():
steps = [
{"op": "selection.face", "params": {}},
{"op": "retouch.skin", "params": {"smooth": 24}},
{"op": "photo.autoTone", "params": {}},
]
guarded = guide.enforce_guards(steps)
assert _ops(guarded) == ["selection.face", "retouch.skin"]
stacked = [
{"op": "adjustments.apply",
"params": {"type": "brightness-contrast", "params": {"brightness": 20}}},
{"op": "adjustments.apply",
"params": {"type": "exposure", "params": {"exposure": 20}}},
]
collapsed = guide.enforce_guards(stacked)
assert len(collapsed) == 1
assert collapsed[0]["params"]["type"] == "shadows-highlights"
def test_plan_route_follows_the_recipe_for_generic_look(client):
import json as _json
eaar.set_editor_planner(lambda messages, owner: _json.dumps({
"reply": "ok",
"steps": [{"op": "adjustments.apply",
"params": {"type": "brightness-contrast", "params": {"brightness": 8}}}],
}))
resp = client.post("/api/editor/plan", json={"prompt": "make it pop"})
assert resp.status_code == 200
data = resp.json()
assert data["steps"][0]["op"] == "photo.autoTone"
assert len(data["steps"]) >= 3
assert data["planner"].endswith("+guide")
def test_guidance_documents_look_and_restore_recipes():
text = guide.guidance()
low = text.lower()
assert "clarity" in low and "vignette" in low
assert "vibrance" in low
assert "restore" in low and "exposure" in low
assert "autoTone" in text
def test_plan_special_format_crops_and_resizes():
special = eaar._plan_special(
"make a 1080x1350 Instagram post from this", {"width": 900, "height": 600}, False)
assert special is not None
steps, reply, note = special
assert _ops(steps) == ["document.crop", "document.resizeCanvas"]
assert "1080" in reply and "1350" in reply
assert set(steps[0]["params"]) >= {"x", "y", "width", "height"}
assert steps[1]["params"]["width"] == 1080 and steps[1]["params"]["height"] == 1350
def test_plan_special_text_add_with_shadow_is_not_generative():
assert eaar._generative_kind("add a title TOKYO with a drop shadow") is None
special = eaar._plan_special("add a title TOKYO with a drop shadow", {}, False)
assert special is not None
steps, reply, note = special
assert _ops(steps) == ["text.add", "layers.addStyle"]
assert steps[0]["params"]["text"] == "TOKYO"
assert steps[1]["params"]["type"] == "drop-shadow"
assert "drop shadow" in reply.lower()
def test_plan_special_extend_resizes_without_provider_generative_with():
steps, reply, note = eaar._plan_special(
"extend the canvas to 16:9", {"width": 900, "height": 600}, False)
assert _ops(steps) == ["document.resizeCanvas"]
assert "provider" in note.lower()
steps2, _, note2 = eaar._plan_special(
"extend the canvas to 16:9", {"width": 900, "height": 600}, True)
assert _ops(steps2) == ["ai.generativeExpand"]
assert note2 == ""
def test_plan_special_psd_export():
special = eaar._plan_special("save as PSD with layers", {}, False)
assert special is not None and _ops(special[0]) == ["document.savePsd"]
def test_background_replacement_is_generative_not_blur():
assert eaar._generative_kind("put her on a beach") == "replace"
assert eaar._generative_kind("remove the background and put her on a beach") == "replace"
def test_verified_reply_corrects_a_false_size_claim():
corrected = eaar._verified_reply(
"make a 1080x1350 Instagram post",
[{"op": "history.mark", "params": {}}],
[{"ok": True}],
{"width": 600, "height": 600},
"Exported your 900x600 as a 1080x1350 Instagram post.",
)
assert "600" in corrected
assert "still" in corrected.lower() or "didn't apply" in corrected.lower()
passed = eaar._verified_reply(
"make a 1080x1350 Instagram post",
[{"op": "document.crop", "params": {}}, {"op": "document.resizeCanvas", "params": {}}],
[{"ok": True}, {"ok": True}],
{"width": 1080, "height": 1350},
"Setting the canvas to 1080\u00d71350.",
)
assert "1080" in passed and "1350" in passed
def test_plan_endpoint_returns_deterministic_format_steps(client, monkeypatch):
eaar.set_editor_planner(lambda messages, owner: {"reply": "Exporting.", "steps": []})
monkeypatch.setattr(eaar, "_image_provider_available", lambda owner: False)
resp = client.post("/api/editor/plan", json={
"prompt": "make a 1080x1350 Instagram post",
"state": {"width": 900, "height": 600},
})
data = resp.json()
assert data["planner"] == "deterministic"
assert _ops(data["steps"])[:2] == ["document.crop", "document.resizeCanvas"]
def test_style_guide_serves_format_text_psd_rules():
rules = guide.as_json()["rules"]
assert "format" in rules and "text" in rules and "psd" in rules
assert guide.STYLE_GUIDE_VERSION >= 3
# ── Quick Edit coverage: structural ops (server planner gaps) ──────────
# Every prompt that failed the live Quick Edit coverage audit because the
# server planner emitted a generic adjustments/transform step instead of the
# exact structural op the browser supports.
_STRUCTURAL_PROMPTS = [
("fill the white area with red", "fill.flood"), # 1
("flood fill the sky with blue", "fill.flood"), # 2
("make the text bigger", "text.edit"), # 3
("make the title red", "text.edit"), # 4
("add a colour overlay", "layers.addStyle"), # 8
("align the logo to the centre", "layers.align"), # 9
("align the layer to the left", "layers.align"), # 10
("rotate only this layer 90 degrees", "layer.rotate"), # 11
("flip this layer horizontally", "layer.flip"), # 12
("rotate the whole image 90 degrees", "document.rotate"), # 13
("flip the canvas vertically", "document.flip"), # 14
("copy the selection", "selection.copy"), # 19
("paste", "selection.paste"), # 20
("flatten the image", "layers.flatten"), # 40
("merge the layer down", "layers.mergeDown"), # 41
]
@pytest.mark.parametrize("prompt,expected_op", _STRUCTURAL_PROMPTS)
def test_structural_plan_routes_to_the_exact_op(prompt, expected_op):
plan = guide.structural_plan(prompt)
assert plan is not None, prompt
assert plan["planner"] == "deterministic"
assert expected_op in _ops(plan["steps"]), (prompt, _ops(plan["steps"]))
@pytest.mark.parametrize("prompt,expected_op", _STRUCTURAL_PROMPTS)
def test_plan_special_emits_structural_ops(prompt, expected_op):
# The server override path (_plan_special) must return the structural step,
# otherwise a wrong model plan suppresses the browser fallback.
special = eaar._plan_special(prompt, {"width": 900, "height": 600}, True)
assert special is not None, prompt
steps, reply, _ = special
assert expected_op in _ops(steps), (prompt, _ops(steps))
assert reply
def test_structural_ops_are_in_the_server_op_catalog():
for name in ("fill.flood", "selection.fill", "text.edit", "layers.align",
"layer.rotate", "layer.flip", "document.rotate", "document.flip",
"selection.copy", "selection.paste", "layers.flatten",
"layers.mergeDown"):
assert name in eaar._OP_SET, name
def test_structural_plan_ignores_non_structural_intents():
for prompt in ("make it brighter", "blur the background", "make it punchy",
"restore this old photo", "smooth the skin", "sharpen it",
"add the title \"Summer\"", "duplicate this layer"):
assert guide.structural_plan(prompt) is None, prompt
def test_structural_params_match_the_client():
# Canonical params must match static/js/editor/galleryEditor.js control ops.
fill = guide.structural_plan("fill the white area with red")["steps"][0]
assert fill["op"] == "fill.flood" and fill["params"]["color"] == "#ff0000"
sel = guide.structural_plan("fill the selection with blue")["steps"][0]
assert sel["op"] == "selection.fill" and sel["params"]["color"] == "#0000ff"
align = guide.structural_plan("align the layer to the left")["steps"][0]
assert align["params"]["mode"] == "left"
rot = guide.structural_plan("rotate only this layer 90 degrees")["steps"][0]
assert rot["params"]["degrees"] == 90
flip = guide.structural_plan("flip the canvas vertically")["steps"][0]
assert flip["op"] == "document.flip" and flip["params"]["axis"] == "v"
overlay = guide.structural_plan("add a colour overlay")["steps"][0]
assert overlay["params"]["type"] == "color-overlay"
edit = guide.structural_plan("make the text bigger")["steps"][0]
assert edit["op"] == "text.edit" and edit["params"]["size"] > 0
def test_plan_route_returns_structural_steps(client):
resp = client.post("/api/editor/plan", json={
"prompt": "flatten the image",
"state": {"width": 900, "height": 600},
})
data = resp.json()
assert data["planner"] == "deterministic"
assert "layers.flatten" in _ops(data["steps"])
def test_style_guide_serves_structural_rule():
rules = guide.as_json()["rules"]
assert "structural" in rules
assert guide.STYLE_GUIDE_VERSION >= 5
# ── Targeted recolour + named reference looks ──────────────────────────
_RECOLOR_CASES = [
("make only the red car blue", "red"),
("recolour the red rock to teal", "red"),
("change the yellow car to red", "yellow"),
]
@pytest.mark.parametrize("prompt,source", _RECOLOR_CASES)
def test_targeted_recolor_selects_object_then_masked_hue(prompt, source):
plan = guide.recolor_plan(prompt)
assert plan is not None
assert _ops(plan["steps"]) == ["selection.samText", "adjustments.applyMasked"]
sel, adj = plan["steps"]
assert sel["params"]["text"] # object description for the mask
assert sel["params"]["fallbackColor"] == guide.color_for_name(source)
assert adj["params"]["mode"] == "hue-saturation"
assert "hue" in adj["params"] and isinstance(adj["params"]["hue"], int)
def test_recolor_never_emits_selection_fill_or_a_global_tint():
for prompt, _ in _RECOLOR_CASES:
ops = _ops(guide.recolor_plan(prompt)["steps"])
# A flat fill destroys shading; a whole-image tint is not targeted.
assert "selection.fill" not in ops
assert "adjustments.apply" not in ops # exact op, not applyMasked
def test_targeted_recolor_hue_delta_is_source_to_target():
# red (hue 0) -> blue (hue 240): shortest rotation is -120 degrees.
plan = guide.recolor_plan("make only the red car blue")
assert plan["steps"][1]["params"]["hue"] == -120
def test_recolor_masked_hue_carries_source_hue():
# red has hue 0; the client uses sourceHue as the rotation center so
# neighbouring colours (chrome, windows, highlights) keep their own hue.
plan = guide.recolor_plan("make only the red car blue")
assert plan["steps"][1]["params"]["sourceHue"] == 0
def test_enforce_guards_drops_global_lift_after_masked_recolor():
# A recolour keeps shading; a later whole-image tone lift would tint the
# rest of the photo, so enforce_guards must drop global lifts after it.
steps = [
{"op": "selection.samText", "params": {"text": "red car"}},
{"op": "adjustments.applyMasked",
"params": {"mode": "hue-saturation", "hue": -120, "sourceHue": 0}},
{"op": "adjustments.apply",
"params": {"type": "brightness-contrast", "params": {"brightness": 20}}},
{"op": "photo.autoTone", "params": {}},
]
assert _ops(guide.enforce_guards(steps)) == ["selection.samText", "adjustments.applyMasked"]
def test_enforce_guards_drops_whole_image_grade_after_masked_recolor():
# Not just tone "lifts": ANY whole-image adjustment (vibrance/saturation/
# autoTone) after a masked recolour re-grades the photo and undoes the
# "only this object" intent, so enforce_guards must drop them all.
steps = [
{"op": "selection.samText", "params": {"text": "red car"}},
{"op": "adjustments.applyMasked",
"params": {"mode": "hue-saturation", "hue": -120, "sourceHue": 0}},
{"op": "adjustments.apply",
"params": {"type": "vibrance", "params": {"vibrance": 20}}},
{"op": "adjustments.apply",
"params": {"type": "hue-saturation", "params": {"saturation": 12}}},
{"op": "photo.autoTone", "params": {}},
]
assert _ops(guide.enforce_guards(steps)) == ["selection.samText", "adjustments.applyMasked"]
def test_enforce_guards_keeps_grades_when_there_is_no_recolor():
# A recipe/look (no masked recolour) still keeps its whole-image grades.
steps = [
{"op": "photo.autoTone", "params": {}},
{"op": "adjustments.apply", "params": {"type": "vibrance", "params": {"vibrance": 18}}},
{"op": "adjustments.apply",
"params": {"type": "brightness-contrast", "params": {"brightness": 6, "contrast": -10}}},
]
assert _ops(guide.enforce_guards(steps)) == ["photo.autoTone", "adjustments.apply"]
def test_recolor_to_white_uses_the_image_model_not_a_hue_shift():
# A white/black/grey target cannot be reached by a hue rotation.
plan = guide.recolor_plan("make the red car white")
assert plan is not None
assert _ops(plan["steps"]) == ["selection.samText", "ai_edit_image"]
assert plan["steps"][1]["params"]["mode"] == "masked"
def test_recolor_with_single_colour_uses_masked_image_model():
plan = guide.recolor_plan("make the flowers bright pink")
assert plan is not None
assert _ops(plan["steps"]) == ["selection.samText", "ai_edit_image"]
assert plan["steps"][0]["params"]["text"] == "flowers"
params = plan["steps"][1]["params"]
assert params["mode"] == "masked"
assert params["steps"] == 8
def test_recolor_does_not_hijack_unrelated_colour_prompts():
for prompt in ("make it blue", "fill the white area with red",
"make the title red", "make this black and white",
"blur the background"):
assert guide.recolor_plan(prompt) is None
@pytest.mark.parametrize("prompt", [
"make it look like a Wes Anderson film",
"make it look like a film noir",
"make it look like vintage Kodachrome",
])
def test_named_recipe_looks_are_adjustment_stacks(prompt):
plan = guide.look_plan(prompt)
assert plan is not None
ops = _ops(plan["steps"])
assert ops and "ai_edit_image" not in ops
@pytest.mark.parametrize("prompt", [
"make it look like cyberpunk",
"make it look like a Studio Ghibli film",
])
def test_strong_named_looks_use_instruction_mode(prompt):
plan = guide.look_plan(prompt)
assert plan is not None
assert _ops(plan["steps"]) == ["ai_edit_image"]
assert plan["steps"][0]["params"]["mode"] == "instruction"
assert plan["steps"][0]["params"]["steps"] == 8
def test_structural_plan_dispatches_looks_and_recolor():
wes = guide.structural_plan("make it look like a Wes Anderson film")
assert "adjustments.apply" in _ops(wes["steps"])
recolor = guide.structural_plan("make only the red car blue")
assert _ops(recolor["steps"]) == ["selection.samText", "adjustments.applyMasked"]
def test_named_looks_do_not_hijack_generative_lookalikes():
for prompt in ("make it look snowy", "make it look like summer"):
assert guide.look_plan(prompt) is None
def test_plan_route_returns_targeted_recolor(client):
resp = client.post("/api/editor/plan", json={
"prompt": "make only the red car blue",
"state": {"width": 900, "height": 600},
})
assert resp.status_code == 200
data = resp.json()
assert data["planner"] == "deterministic"
assert _ops(data["steps"]) == ["selection.samText", "adjustments.applyMasked"]
assert "selection.fill" not in _ops(data["steps"])
def test_plan_route_returns_named_look(client):
resp = client.post("/api/editor/plan", json={
"prompt": "make it look like a Wes Anderson film",
"state": {"width": 900, "height": 600},
})
assert resp.status_code == 200
data = resp.json()
assert data["planner"] == "deterministic"
assert "adjustments.apply" in _ops(data["steps"])
def test_style_guide_serves_recolor_and_looks_rules():
body = guide.as_json()
assert "recolor" in body["rules"]
assert "looks" in body["rules"]
assert {lk["id"] for lk in body["looks"]} >= {
"wes_anderson", "film_noir", "cyberpunk", "vintage_kodachrome",
"studio_ghibli",
}
if __name__ == "__main__": # pragma: no cover
raise SystemExit(pytest.main([__file__, "-q"]))
+51
View File
@@ -0,0 +1,51 @@
"""Graceful-shutdown signalling tests (HARNESS-owned).
SSE / streaming responses never complete on their own, so uvicorn's graceful
shutdown would wait for an open browser stream up to the whole
``timeout_graceful_shutdown`` window. src.shutdown is a process-wide flag that
the SIGTERM/SIGINT handler flips and every streaming generator checks, so a
restart ends open streams promptly while ordinary in-flight requests still
finish.
"""
from pathlib import Path
from src import shutdown
ROOT = Path(__file__).resolve().parent.parent
def test_request_shutdown_is_sticky_and_resettable():
shutdown.reset()
try:
assert shutdown.is_shutting_down() is False
shutdown.request_shutdown()
assert shutdown.is_shutting_down() is True
# idempotent
shutdown.request_shutdown()
assert shutdown.is_shutting_down() is True
finally:
shutdown.reset()
assert shutdown.is_shutting_down() is False
def test_app_installs_its_own_shutdown_handler():
src = (ROOT / "app.py").read_text(encoding="utf-8")
# This uvicorn version has no install_signal_handlers kwarg, so app.py wraps
# server.handle_exit (which capture_signals wires SIGINT/SIGTERM to).
assert "install_signal_handlers=False" not in src
assert "server.handle_exit = _handle_shutdown_signal" in src
assert "_shutdown.request_shutdown()" in src
assert "server.run()" in src
def test_streaming_generators_check_the_shutdown_flag():
# editor bridge SSE + chat stream/resume + research + email events + shell.
for rel in (
"routes/editor_agent_routes.py",
"routes/chat_routes.py",
"routes/research/research_routes.py",
"routes/email/email_routes.py",
"routes/shell_routes.py",
):
src = (ROOT / rel).read_text(encoding="utf-8")
assert "_shutdown.is_shutting_down()" in src, rel
+14 -2
View File
@@ -21,7 +21,7 @@ described as a switch that turns something off, the read rejects `0`, `false`,
`no` and `off` and treats everything else as on. The `Default` column is the
value the code falls back to when the variable is unset, quoted from the source.
The source tree reads **117** `ODYSSEUS_*` variables: 81 an operator may want to set, and 36 that are internal - sentinels, fixture switches, capture hooks and development tooling. The internal ones are listed too, in their own section, so this page can be checked against the source mechanically.
The source tree reads **124** `ODYSSEUS_*` variables: 87 an operator may want to set, and 37 that are internal - sentinels, fixture switches, capture hooks and development tooling. The internal ones are listed too, in their own section, so this page can be checked against the source mechanically.
> This page is generated. Edit `scripts/generate_env_reference.py` and
> re-run it; `tests/test_env_reference.py` enforces that the committed page
@@ -58,10 +58,13 @@ The source tree reads **117** `ODYSSEUS_*` variables: 81 an operator may want to
| Variable | Default | Read in | What it does |
|---|---|---|---|
| `ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS` | `''` | `routes/gallery/gallery_routes.py` | Set to 1, true, yes or on to allow local image-generation and editing backends to be reachable as image providers. |
| `ODYSSEUS_COPILOT_API_VERSION` | `'2026-06-01'` | `src/copilot.py` | Dated API-version header the Copilot models and chat endpoints require. |
| `ODYSSEUS_COPILOT_CLIENT_ID` | `'01ab8ac9400c4e429b23'` | `src/copilot.py` | GitHub OAuth client id for the Copilot device flow. The default is the public VS Code client id; override it only with your own allow-listed app. |
| `ODYSSEUS_DEEPSEEK_REASONING_EFFORT` | `'high'` | `src/llm_core.py` | Reasoning effort for DeepSeek. Only `high` and `max` are accepted; any other value falls back to the default. |
| `ODYSSEUS_EDITOR_AGENT_BASES` | *unset* | `src/editor_agent_model.py` | Comma-separated base URLs for the image-editor agent backends; a single base URL in the model endpoint config wins over this list. |
| `ODYSSEUS_FIRST_TOKEN_TIMEOUT` | `''` | `src/llm_core.py` | Seconds to wait for the first streamed token from a local endpoint before failing. Unset keeps the generous read timeout, which makes a stalled backend look like a hung agent. |
| `ODYSSEUS_IMAGE_OPS_BASE` | `''` | `routes/gallery/gallery_routes.py` | Base URL of the image-operations backend (inpaint, outpainting and style transfer); a trailing slash is stripped before use. |
| `ODYSSEUS_LOCAL_MODEL_GATE` | `'true'` | `src/llm_core.py` | On by default. Set 0, false, no or off to drop the gate that checks a local endpoint before routing a request to it. |
| `ODYSSEUS_MISTRAL_REASONING_EFFORT` | `'high'` | `src/llm_core.py` | Reasoning effort sent to Mistral thinking-capable models. The API accepts high, medium, low and none. |
| `ODYSSEUS_MLX_IMAGE_VLM_MODEL` | *unset* | `scripts/mlx_image_server.py` | Vision-language model id for the MLX image server script. Required unless `--vlm-model` is passed on the command line. |
@@ -126,6 +129,9 @@ The source tree reads **117** `ODYSSEUS_*` variables: 81 an operator may want to
| `ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES` | `25 * 1024 * 1024` | `src/upload_limits.py` (+1 more) | Maximum bytes accepted for an image handed to a gallery transform. |
| `ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES` | `100 * 1024 * 1024` | `src/upload_limits.py` (+1 more) | Maximum bytes accepted for a gallery upload. |
| `ODYSSEUS_ICS_MAX_BYTES` | `10 * 1024 * 1024` | `src/upload_limits.py` | Maximum bytes accepted for an imported ICS file. |
| `ODYSSEUS_IMAGE_EDIT_MAX_SIDE` | `'768'` | `routes/gallery/gallery_routes.py` | Maximum pixel side (width or height) an edited image is downscaled to before it is sent to the image backend. |
| `ODYSSEUS_IMAGE_EDIT_MAX_STEPS` | `'8'` | `routes/gallery/gallery_routes.py` | Maximum number of iterative edit steps the image backend may perform in a single request. |
| `ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT` | `'900'` | `routes/gallery/gallery_routes.py` | Seconds the outpainting proxy waits for the image backend before giving up on a single request. |
| `ODYSSEUS_MEDIA_FRAME_TIMEOUT` | `30` | `src/media_ingress.py` | Seconds allowed for extracting frames from a video before giving up. |
| `ODYSSEUS_MEDIA_MAX_AUDIO_BYTES` | `32 * 1024 * 1024` | `src/media_ingress.py` | Largest source audio file the media pipeline will read. |
| `ODYSSEUS_MEDIA_MAX_DIMENSION` | `1600` | `src/media_ingress.py` | Longest edge in pixels an image is resized down to before encoding. |
@@ -184,6 +190,12 @@ The source tree reads **117** `ODYSSEUS_*` variables: 81 an operator may want to
Listed for completeness. Setting one of these on a real install is either a no-op or a way to break something quietly.
### Deployment and first run
| Variable | Default | Read in | What it does |
|---|---|---|---|
| `ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT` | `'900'` | `app.py` | Seconds the server waits for in-flight requests and background loops to finish during a graceful shutdown. |
### Model routing and providers
| Variable | Default | Read in | What it does |
@@ -265,7 +277,7 @@ reads three ways, because no single pattern covers the codebase:
lines, so one read lives inside a string literal.
The three passes are not redundancy. A line-based grep for a direct
`os.environ.get("ODYSSEUS_...` call finds 87 of the 117 variables on this
`os.environ.get("ODYSSEUS_...` call finds 94 of the 124 variables on this
page. What it misses is reads through an env-reader helper, reads whose call
spans more than one line, reads whose variable name is held in a module
constant, and reads through a mapping passed in as an argument - which is the