mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-16 20:02:20 +02:00
feat(photo-editor): NL editor planner, image ops, email/gallery server
Server-side photo editor: natural-language editor agent route
(/api/editor/agent), photo_editor tool family + style guide, image-op
provider proxy with progress, email server hardening, gallery/hwfit
routes, and graceful shutdown.
Source: vendor/photocraft (ad1cff44) editor-server files, merged onto
upstream/dev (a8c147b2). Conflict resolution documented in
pr_work/B_conflicts.md.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
pewdiepie-archdaemon
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
c74f475679
commit
eb940f24a3
@@ -0,0 +1,61 @@
|
||||
"""In-flight activity counters (heavy image ops) for safe restarts/observability.
|
||||
|
||||
A deploy restart kills any in-flight long image op (a diffusion edit is 60-120 s),
|
||||
so the operator needs to know whether the process is busy before issuing
|
||||
``systemctl --user restart``. These counters are process-local and cheap.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from typing import Dict
|
||||
|
||||
_lock = threading.Lock()
|
||||
_image_ops = 0
|
||||
_image_ops_by_name: Dict[str, int] = {}
|
||||
_started_at = time.time()
|
||||
|
||||
|
||||
def begin_image_op(name: str = "image") -> None:
|
||||
global _image_ops
|
||||
with _lock:
|
||||
_image_ops += 1
|
||||
_image_ops_by_name[name] = _image_ops_by_name.get(name, 0) + 1
|
||||
|
||||
|
||||
def end_image_op(name: str = "image") -> None:
|
||||
global _image_ops
|
||||
with _lock:
|
||||
_image_ops = max(0, _image_ops - 1)
|
||||
if name in _image_ops_by_name:
|
||||
_image_ops_by_name[name] -= 1
|
||||
if _image_ops_by_name[name] <= 0:
|
||||
_image_ops_by_name.pop(name, None)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def image_op(name: str = "image"):
|
||||
begin_image_op(name)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
end_image_op(name)
|
||||
|
||||
|
||||
def activity() -> Dict[str, object]:
|
||||
with _lock:
|
||||
return {
|
||||
"image_ops": _image_ops,
|
||||
"image_ops_by_name": dict(_image_ops_by_name),
|
||||
"uptime_s": round(time.time() - _started_at, 1),
|
||||
}
|
||||
|
||||
|
||||
def heavy_image_ops_active() -> int:
|
||||
with _lock:
|
||||
return _image_ops
|
||||
|
||||
|
||||
def busy() -> bool:
|
||||
return heavy_image_ops_active() > 0
|
||||
@@ -0,0 +1,646 @@
|
||||
"""In-process agent tools for the scoped ``photo_editor`` capability family.
|
||||
|
||||
These handlers run as the logged-in owner and drive the image editor over the
|
||||
owner-scoped in-process bridge (``/api/editor/bridge/*``) plus the app's
|
||||
existing image transforms (``/api/image/*``). They are only offered to the
|
||||
model when the ``photo_editor`` family is active (the Quick Edit popup or an
|
||||
explicit photo-edit chat turn), never in the general tool list.
|
||||
|
||||
Loopback calls carry the internal token and impersonate ``owner`` (middleware
|
||||
app.py), so the editor bridge resolves the same ``owner:session`` scope the
|
||||
browser subscriber uses.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from src.tools._common import _INTERNAL_BASE, _internal_headers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _provider_message() -> str:
|
||||
try:
|
||||
from src.ai_interaction import IMAGE_PROVIDER_REQUIRED_MESSAGE
|
||||
|
||||
return IMAGE_PROVIDER_REQUIRED_MESSAGE
|
||||
except Exception:
|
||||
return (
|
||||
"Removing/adding objects needs an image generation provider; enable one "
|
||||
"in Settings > Models."
|
||||
)
|
||||
|
||||
|
||||
def _provider_available(owner: Optional[str]) -> bool:
|
||||
try:
|
||||
from src.ai_interaction import image_editing_provider_available
|
||||
|
||||
return bool(image_editing_provider_available(owner))
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
# Local diffusion/image servers can take 90-110s per image (8 steps) and
|
||||
# longer on a cold start; the previous 120s default timed out mid-generation.
|
||||
_EDITOR_TIMEOUT = 300.0
|
||||
# A mask covering less than this fraction of the image cannot drive a useful
|
||||
# regional inpaint (tiny figures, empty SAM output): fall back to grounding or
|
||||
# whole-image instruction mode instead.
|
||||
_MIN_MASK_FRACTION = 0.002
|
||||
_AI_OPS = {
|
||||
"remove-bg": "/api/image/remove-bg",
|
||||
"upscale": "/api/image/upscale-local",
|
||||
"denoise": "/api/image/denoise",
|
||||
"enhance-face": "/api/image/enhance-face",
|
||||
"inpaint": "/api/image/inpaint",
|
||||
"generative": "/api/image/inpaint",
|
||||
}
|
||||
_PLACE_OP = "layers.placeImage"
|
||||
|
||||
|
||||
def _parse(content: Any) -> Dict[str, Any]:
|
||||
raw = content if isinstance(content, str) else json.dumps(content or {})
|
||||
try:
|
||||
parsed = json.loads(raw) if raw.strip() else {}
|
||||
except (ValueError, TypeError):
|
||||
parsed = {}
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
|
||||
|
||||
def _session(args: Dict[str, Any], ctx: Dict[str, Any]) -> str:
|
||||
return str(args.get("session") or args.get("editor_session") or ctx.get("session_id") or "default")
|
||||
|
||||
|
||||
async def _post(path: str, payload: Dict[str, Any], owner: Optional[str], timeout: float = _EDITOR_TIMEOUT) -> Dict[str, Any]:
|
||||
headers = {**_internal_headers(owner), "Content-Type": "application/json"}
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
resp = await client.post(f"{_INTERNAL_BASE}{path}", json=payload, headers=headers)
|
||||
resp.raise_for_status()
|
||||
try:
|
||||
return resp.json()
|
||||
except ValueError:
|
||||
return {"raw": resp.text}
|
||||
|
||||
|
||||
def _as_data_url(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
if value.startswith("data:"):
|
||||
return value
|
||||
if value.startswith("/"):
|
||||
return None
|
||||
return f"data:image/png;base64,{value}"
|
||||
|
||||
|
||||
def _data_url_bytes(data_url: str) -> bytes:
|
||||
_, _, b64 = data_url.partition(",")
|
||||
return base64.b64decode(b64)
|
||||
|
||||
|
||||
def _persist_png(image_bytes: bytes, owner: Optional[str], session_id: Optional[str], prompt: str, model: str) -> Dict[str, Any]:
|
||||
"""Save an edited/generated PNG and return {image_id, image_url, image_size}."""
|
||||
from core.database import GalleryImage, SessionLocal
|
||||
from src.constants import GENERATED_IMAGES_DIR
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
root = Path(GENERATED_IMAGES_DIR).resolve()
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
filename = f"{uuid.uuid4().hex[:12]}.png"
|
||||
(root / filename).write_bytes(image_bytes)
|
||||
new_id = str(uuid.uuid4())
|
||||
width = height = 0
|
||||
try:
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(io.BytesIO(image_bytes)) as img:
|
||||
width, height = img.size
|
||||
except Exception:
|
||||
pass
|
||||
size = f"{width}x{height}" if width and height else ""
|
||||
# The editor bridge uses an owner-scoped session id (e.g. "pewds:3")
|
||||
# that is not a chat-sessions row. GalleryImage.session_id is a FK to
|
||||
# sessions.id, so drop an unknown id instead of raising IntegrityError.
|
||||
if session_id:
|
||||
from core.database import Session as _ChatSession
|
||||
|
||||
try:
|
||||
known = db.query(_ChatSession.id).filter(_ChatSession.id == session_id).first()
|
||||
except Exception:
|
||||
known = None
|
||||
if known is None:
|
||||
session_id = None
|
||||
derived = GalleryImage(
|
||||
id=new_id,
|
||||
filename=filename,
|
||||
prompt=prompt or None,
|
||||
caption=None,
|
||||
model=model,
|
||||
size=size or None,
|
||||
quality=None,
|
||||
tags=None,
|
||||
ai_tags=None,
|
||||
session_id=session_id,
|
||||
album_id=None,
|
||||
owner=owner,
|
||||
file_hash=hashlib.sha256(image_bytes).hexdigest(),
|
||||
file_size=len(image_bytes),
|
||||
width=width or None,
|
||||
height=height or None,
|
||||
)
|
||||
db.add(derived)
|
||||
db.commit()
|
||||
return {"image_id": new_id, "image_url": f"/api/generated-image/{filename}", "image_size": size or None}
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def _place_layer(data_url: str, name: str, owner: Optional[str], session: str, *, masked: bool = False, blend: str = "normal", opacity: float = 100.0, mode: Optional[str] = None) -> bool:
|
||||
params: Dict[str, Any] = {
|
||||
"dataUrl": data_url,
|
||||
"name": name,
|
||||
"masked": bool(masked),
|
||||
"blend": blend or "normal",
|
||||
"opacity": opacity,
|
||||
}
|
||||
# Forward the edit mode so the client can build a difference mask for
|
||||
# instruction-mode results: the whole image was regenerated under an
|
||||
# instruction prompt, so untouched pixels must stay pixel-identical (the
|
||||
# client keeps only the changed region) instead of the new layer overwriting
|
||||
# the entire canvas. Omit the keys entirely when no mode is given so the
|
||||
# client keeps its current behavior (e.g. generate_image must not diff-mask).
|
||||
if mode:
|
||||
params["mode"] = mode
|
||||
params["differenceMask"] = mode == "instruction"
|
||||
if mode == "instruction":
|
||||
# Exclude a thin edge band from the client difference mask: full-frame
|
||||
# regenerations can shift pixels along the border.
|
||||
params["edgeExcludePx"] = 3
|
||||
payload = {
|
||||
"session": session,
|
||||
"op": _PLACE_OP,
|
||||
"params": params,
|
||||
"wait": True,
|
||||
"timeout": 60,
|
||||
}
|
||||
try:
|
||||
data = await _post("/api/editor/bridge/command", payload, owner, timeout=90)
|
||||
return data.get("ok") is not False
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.warning("editor layer placement failed: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
async def _fetch_selection_mask(owner: Optional[str], session: str) -> Optional[str]:
|
||||
"""Return the editor's current selection as a data-URL mask, or None.
|
||||
|
||||
Generative ops (inpaint/generative) need a mask; the planner/judge may not
|
||||
have supplied one, so fall back to whatever is selected in the editor.
|
||||
"""
|
||||
payload = {
|
||||
"session": session,
|
||||
"op": "selection.exportMask",
|
||||
"params": {},
|
||||
"wait": True,
|
||||
"timeout": 30,
|
||||
}
|
||||
try:
|
||||
data = await _post("/api/editor/bridge/command", payload, owner, timeout=45)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.warning("editor selection mask fetch failed: %s", exc)
|
||||
return None
|
||||
result = data.get("result") if isinstance(data, dict) else None
|
||||
if isinstance(result, dict):
|
||||
return _as_data_url(result.get("mask"))
|
||||
return None
|
||||
|
||||
|
||||
def _mask_white_fraction(data_url: Optional[str]) -> float:
|
||||
"""Fraction of a grayscale mask that is selected (white, >=128)."""
|
||||
if not data_url:
|
||||
return 0.0
|
||||
try:
|
||||
from PIL import Image as _PILImage
|
||||
|
||||
with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))).convert("L") as im:
|
||||
total = im.width * im.height
|
||||
if not total:
|
||||
return 0.0
|
||||
hist = im.histogram()
|
||||
return sum(hist[128:]) / total
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _image_is_blank(data_url: str) -> bool:
|
||||
"""True when a result has an alpha channel that is essentially transparent."""
|
||||
try:
|
||||
from PIL import Image as _PILImage
|
||||
|
||||
with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))) as im:
|
||||
if im.mode not in ("RGBA", "LA") and "transparency" not in im.info:
|
||||
return False
|
||||
alpha = im.convert("RGBA").getchannel("A")
|
||||
hist = alpha.histogram()
|
||||
total = sum(hist)
|
||||
return total > 0 and (sum(hist[1:]) / total) < 0.001
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _rect_mask_data_url(image_data_url: str, box: List[float], *, pad_frac: float = 0.15) -> Optional[str]:
|
||||
"""Build a rectangular mask data URL from a [x1,y1,x2,y2] box, dilated.
|
||||
|
||||
Used when the grounding detector finds an object but SAM returns an empty
|
||||
mask (tiny figures): a dilated box guarantees a usable region.
|
||||
"""
|
||||
try:
|
||||
from PIL import Image as _PILImage, ImageDraw
|
||||
|
||||
with _PILImage.open(io.BytesIO(_data_url_bytes(image_data_url))) as im:
|
||||
w, h = im.size
|
||||
x1, y1, x2, y2 = (float(v) for v in box)
|
||||
bw, bh = x2 - x1, y2 - y1
|
||||
x1, y1 = max(0.0, x1 - bw * pad_frac), max(0.0, y1 - bh * pad_frac)
|
||||
x2, y2 = min(float(w), x2 + bw * pad_frac), min(float(h), y2 + bh * pad_frac)
|
||||
mask = _PILImage.new("L", (w, h), 0)
|
||||
ImageDraw.Draw(mask).rectangle([x1, y1, x2, y2], fill=255)
|
||||
buf = io.BytesIO()
|
||||
mask.save(buf, "PNG")
|
||||
return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def _ground_box(owner: Optional[str], image_data_url: str, label: str) -> Optional[List[float]]:
|
||||
"""Use the in-app grounding detector (OWL-ViT via /api/image/mask) for a box."""
|
||||
try:
|
||||
data = await _post("/api/image/mask", {"image": image_data_url, "text": label}, owner, timeout=90)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.warning("editor grounding box failed: %s", exc)
|
||||
return None
|
||||
box = data.get("bbox") if isinstance(data, dict) else None
|
||||
if isinstance(box, (list, tuple)) and len(box) == 4:
|
||||
try:
|
||||
return [float(v) for v in box]
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_keep_clause(text: str) -> str:
|
||||
"""Append a keep-the-subject clause when the prompt lacks one."""
|
||||
body = str(text or "").strip()
|
||||
if not body:
|
||||
return body
|
||||
if re.search(r"\b(keep|preserve|leave|unchanged|identical|same)\b", body, re.I):
|
||||
return body
|
||||
return body.rstrip(". ") + ". Keep the main subject and everything else in the image exactly the same."
|
||||
|
||||
|
||||
def _mask_label(args: Dict[str, Any], prompt: str) -> str:
|
||||
"""Best-effort object noun for grounding a regional removal."""
|
||||
label = str(args.get("label") or args.get("object") or "").strip()
|
||||
if label:
|
||||
return label
|
||||
m = re.search(
|
||||
r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out)\s+(?:the\s+|a\s+|an\s+)?"
|
||||
r"([a-z][a-z \-]{1,30}?)(?:\s+(?:on|in|at|from|to|near|behind|beside|by)\b|[.,;!?]|$)",
|
||||
prompt or "",
|
||||
re.I,
|
||||
)
|
||||
if m:
|
||||
noun = m.group(1).strip().lower()
|
||||
if noun and noun not in {"background", "bg"}:
|
||||
return noun
|
||||
for noun in ("person", "people", "man", "woman", "car", "dog", "sign", "logo"):
|
||||
if re.search(rf"\b{re.escape(noun)}\b", prompt or "", re.I):
|
||||
return "person" if noun in {"person", "people", "man", "woman"} else noun
|
||||
return "object"
|
||||
|
||||
|
||||
class PlanEditTool:
|
||||
"""Plan an image edit as ordered editor operations (no side effects)."""
|
||||
|
||||
async def execute(self, content: Any, ctx: Dict[str, Any]):
|
||||
args = _parse(content)
|
||||
prompt = str(args.get("prompt") or "").strip()
|
||||
if not prompt:
|
||||
return "plan_edit: invalid", {"error": "plan_edit needs a non-empty `prompt`.", "exit_code": 1}
|
||||
state = args.get("state") if isinstance(args.get("state"), dict) else {}
|
||||
try:
|
||||
data = await _post(
|
||||
"/api/editor/plan",
|
||||
{"prompt": prompt, "state": state, "mode": "plan", "session": _session(args, ctx)},
|
||||
ctx.get("owner"),
|
||||
)
|
||||
except Exception as exc:
|
||||
return "plan_edit: error", {"error": f"plan_edit failed: {exc}", "exit_code": 1}
|
||||
steps = data.get("steps") or []
|
||||
reply = str(data.get("reply") or "")
|
||||
return f"plan_edit: {len(steps)} step(s)", {
|
||||
"output": reply or f"Planned {len(steps)} editor step(s).",
|
||||
"reply": reply,
|
||||
"planner": data.get("planner"),
|
||||
"steps": steps,
|
||||
"exit_code": 0,
|
||||
}
|
||||
|
||||
|
||||
class RunEditorCommandTool:
|
||||
"""Run a single validated editor op in the browser via the owner bridge."""
|
||||
|
||||
async def execute(self, content: Any, ctx: Dict[str, Any]):
|
||||
args = _parse(content)
|
||||
op = str(args.get("op") or "").strip()
|
||||
if not op:
|
||||
return "run_editor_command: invalid", {"error": "run_editor_command needs `op`.", "exit_code": 1}
|
||||
params = args.get("params") if isinstance(args.get("params"), dict) else {}
|
||||
payload = {
|
||||
"session": _session(args, ctx),
|
||||
"op": op,
|
||||
"params": params,
|
||||
"wait": bool(args.get("wait", True)),
|
||||
"timeout": float(args.get("timeout") or 60),
|
||||
}
|
||||
try:
|
||||
data = await _post("/api/editor/bridge/command", payload, ctx.get("owner"))
|
||||
except Exception as exc:
|
||||
return f"run_editor_command: {op}", {"error": f"run_editor_command failed: {exc}", "exit_code": 1}
|
||||
ok = data.get("ok")
|
||||
result = data.get("result")
|
||||
summary = json.dumps({"ok": ok, "error": data.get("error"), "result": result}, ensure_ascii=False)
|
||||
return f"run_editor_command: {op}", {
|
||||
"output": summary[:4000],
|
||||
"result": result,
|
||||
"screenshot": data.get("screenshot"),
|
||||
"ok": ok,
|
||||
"exit_code": 0 if ok is not False else 1,
|
||||
}
|
||||
|
||||
|
||||
class GetEditorScreenshotTool:
|
||||
"""Capture the current editor canvas as a data URL."""
|
||||
|
||||
async def execute(self, content: Any, ctx: Dict[str, Any]):
|
||||
args = _parse(content)
|
||||
payload = {
|
||||
"session": _session(args, ctx),
|
||||
"op": "screenshot",
|
||||
"params": {},
|
||||
"wait": True,
|
||||
"timeout": float(args.get("timeout") or 30),
|
||||
}
|
||||
try:
|
||||
data = await _post("/api/editor/bridge/command", payload, ctx.get("owner"), timeout=45)
|
||||
except Exception as exc:
|
||||
return "get_editor_screenshot: error", {"error": f"get_editor_screenshot failed: {exc}", "exit_code": 1}
|
||||
result = data.get("result") if isinstance(data.get("result"), dict) else {}
|
||||
shot = data.get("screenshot") or result.get("dataUrl")
|
||||
if not shot:
|
||||
return "get_editor_screenshot: unavailable", {
|
||||
"error": "No editor screenshot was returned (is the editor open?).",
|
||||
"exit_code": 1,
|
||||
}
|
||||
return "get_editor_screenshot", {"output": "Captured the editor canvas.", "screenshot": shot, "exit_code": 0}
|
||||
|
||||
|
||||
class AiEditImageTool:
|
||||
"""Apply an AI image transform (remove-bg/upscale/denoise/enhance/inpaint)."""
|
||||
|
||||
async def execute(self, content: Any, ctx: Dict[str, Any]):
|
||||
args = _parse(content)
|
||||
_t = {"start": time.perf_counter()}
|
||||
op = str(args.get("op") or "").strip().lower()
|
||||
if op not in _AI_OPS:
|
||||
return "ai_edit_image: invalid", {
|
||||
"error": f"ai_edit_image `op` must be one of {sorted(_AI_OPS)}.",
|
||||
"exit_code": 1,
|
||||
}
|
||||
owner = ctx.get("owner")
|
||||
if op in ("inpaint", "generative") and not _provider_available(owner):
|
||||
return "ai_edit_image: provider required", {
|
||||
"error": _provider_message(),
|
||||
"failure_kind": "image_provider_unavailable",
|
||||
"needs_provider": True,
|
||||
"exit_code": 1,
|
||||
}
|
||||
session = _session(args, ctx)
|
||||
image = _as_data_url(args.get("image"))
|
||||
if image is None and isinstance(args.get("image"), str) and args["image"].startswith("/"):
|
||||
# A same-origin gallery/attachment URL: fetch it (owner-scoped) as bytes.
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
resp = await client.get(f"{_INTERNAL_BASE}{args['image']}", headers=_internal_headers(owner))
|
||||
resp.raise_for_status()
|
||||
image = _as_data_url("data:image/png;base64," + base64.b64encode(resp.content).decode())
|
||||
except Exception:
|
||||
image = None
|
||||
used_canvas = False
|
||||
if not image:
|
||||
shot = await GetEditorScreenshotTool().execute(json.dumps({"session": session}), ctx)
|
||||
image = (shot[1] or {}).get("screenshot") if isinstance(shot, tuple) else None
|
||||
used_canvas = bool(image)
|
||||
if not image:
|
||||
return "ai_edit_image: no image", {
|
||||
"error": "ai_edit_image needs an `image` (data URL or same-origin URL) or an open editor.",
|
||||
"exit_code": 1,
|
||||
}
|
||||
_t["image"] = time.perf_counter()
|
||||
payload: Dict[str, Any] = {"image": image}
|
||||
model_choice = str(args.get("model") or "").strip()
|
||||
mask_used = False
|
||||
place_mode: Optional[str] = None
|
||||
if op == "upscale":
|
||||
payload["scale"] = int(args.get("scale") or 2)
|
||||
elif op == "denoise":
|
||||
payload["strength"] = float(args.get("strength") or 0.5)
|
||||
elif op in ("inpaint", "generative"):
|
||||
payload["prompt"] = str(args.get("prompt") or "").strip()
|
||||
if model_choice:
|
||||
# inpaint_proxy reads `_model` to route to a specific model.
|
||||
payload["_model"] = model_choice
|
||||
mode = str(args.get("mode") or "instruction").strip().lower()
|
||||
_steps = int(args.get("steps") or 0)
|
||||
if _steps > 0:
|
||||
payload["steps"] = _steps
|
||||
mask_used = False
|
||||
mask = None
|
||||
if mode == "masked":
|
||||
mask = _as_data_url(args.get("mask"))
|
||||
if not mask:
|
||||
mask = await _fetch_selection_mask(owner, session)
|
||||
# An empty/tiny SAM mask on small figures is useless: drop it.
|
||||
if mask and _mask_white_fraction(mask) < _MIN_MASK_FRACTION:
|
||||
mask = None
|
||||
if not mask:
|
||||
# Fall back to the grounding detector (OWL-ViT): a dilated
|
||||
# box mask around the matching object.
|
||||
box = await _ground_box(owner, image, _mask_label(args, payload["prompt"]))
|
||||
if box:
|
||||
candidate = _rect_mask_data_url(image, box, pad_frac=0.15)
|
||||
if candidate and _mask_white_fraction(candidate) >= _MIN_MASK_FRACTION:
|
||||
mask = candidate
|
||||
if mask:
|
||||
payload["mask"] = mask
|
||||
payload["mode"] = "masked"
|
||||
mask_used = True
|
||||
else:
|
||||
# Nothing usable to mask -> whole-image instruction fallback
|
||||
# rather than placing a transparent/empty layer.
|
||||
mode = "instruction"
|
||||
if mode != "masked":
|
||||
# Instruction mode: whole-image edit with no tight mask. Send a
|
||||
# full-white mask so the edit endpoint accepts the request and
|
||||
# regenerates the whole frame under the instruction prompt, which
|
||||
# names what must stay unchanged. This avoids the mask-compositing
|
||||
# halos that loose selections caused.
|
||||
payload["mode"] = "instruction"
|
||||
payload.pop("mask", None)
|
||||
payload["prompt"] = _ensure_keep_clause(payload["prompt"])
|
||||
try:
|
||||
from PIL import Image as _PILImage
|
||||
|
||||
_raw = _data_url_bytes(image)
|
||||
_w, _h = _PILImage.open(io.BytesIO(_raw)).size
|
||||
_mbuf = io.BytesIO()
|
||||
_PILImage.new("L", (_w, _h), 255).save(_mbuf, "PNG")
|
||||
payload["mask"] = "data:image/png;base64," + base64.b64encode(_mbuf.getvalue()).decode()
|
||||
except Exception:
|
||||
pass
|
||||
place_mode = "masked" if mask_used else "instruction"
|
||||
_t["prep"] = time.perf_counter()
|
||||
try:
|
||||
data = await _post(_AI_OPS[op], payload, owner)
|
||||
except Exception as exc:
|
||||
logger.info(
|
||||
"image-op ai_edit_image op=%s stage=provider FAILED image_ms=%.1f prep_ms=%.1f total_ms=%.1f err=%s",
|
||||
op, (_t["image"] - _t["start"]) * 1000.0, (_t["prep"] - _t["image"]) * 1000.0,
|
||||
(time.perf_counter() - _t["start"]) * 1000.0, exc,
|
||||
)
|
||||
return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) failed: {exc}", "exit_code": 1}
|
||||
_t["provider"] = time.perf_counter()
|
||||
result = data.get("image") or data.get("data")
|
||||
data_url = _as_data_url(result)
|
||||
if not data_url:
|
||||
return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) returned no image.", "exit_code": 1}
|
||||
if _image_is_blank(data_url):
|
||||
return f"ai_edit_image: {op}", {
|
||||
"error": f"ai_edit_image ({op}) produced an empty layer.",
|
||||
"failure_kind": "empty_result",
|
||||
"exit_code": 1,
|
||||
}
|
||||
|
||||
info: Dict[str, Any] = {
|
||||
"output": f"Applied AI edit '{op}'" + (" to the current canvas." if used_canvas else "."),
|
||||
"op": op,
|
||||
"model": data.get("model") or model_choice or "",
|
||||
"exit_code": 0,
|
||||
}
|
||||
try:
|
||||
info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), f"ai_edit_image:{op}", f"ai_edit_image:{op}"))
|
||||
except Exception as exc:
|
||||
logger.warning("ai_edit_image persist failed: %s", exc)
|
||||
_t["persist"] = time.perf_counter()
|
||||
if bool(args.get("place", True)):
|
||||
placed = await _place_layer(
|
||||
data_url,
|
||||
f"AI {op}",
|
||||
owner,
|
||||
session,
|
||||
masked=mask_used,
|
||||
blend=str(args.get("blend") or "normal"),
|
||||
opacity=float(args.get("opacity") or 100),
|
||||
mode=place_mode,
|
||||
)
|
||||
info["placed"] = placed
|
||||
if placed:
|
||||
info["output"] += " Placed the result as a new layer."
|
||||
_t["place"] = time.perf_counter()
|
||||
logger.info(
|
||||
"image-op ai_edit_image op=%s mode=%s canvas=%s image_ms=%.1f mask_ms=%.1f "
|
||||
"model_ms=%.1f persist_ms=%.1f place_ms=%.1f total_ms=%.1f",
|
||||
op, place_mode or "instruction", used_canvas,
|
||||
(_t["image"] - _t["start"]) * 1000.0,
|
||||
(_t["prep"] - _t["image"]) * 1000.0,
|
||||
(_t["provider"] - _t["prep"]) * 1000.0,
|
||||
(_t["persist"] - _t["provider"]) * 1000.0,
|
||||
(_t["place"] - _t["persist"]) * 1000.0,
|
||||
(_t["place"] - _t["start"]) * 1000.0,
|
||||
)
|
||||
return f"ai_edit_image: {op}", info
|
||||
|
||||
|
||||
class GenerateImageLayerTool:
|
||||
"""Generate an image with the configured image model and place it as a layer."""
|
||||
|
||||
async def execute(self, content: str, ctx: dict):
|
||||
args = _parse(content)
|
||||
prompt = str(args.get("prompt") or "").strip()
|
||||
if not prompt:
|
||||
return "generate_image: invalid", {"error": "generate_image needs a non-empty `prompt`.", "exit_code": 1}
|
||||
owner = ctx.get("owner")
|
||||
if not _provider_available(owner):
|
||||
return "generate_image: provider required", {
|
||||
"error": _provider_message(),
|
||||
"failure_kind": "image_provider_unavailable",
|
||||
"needs_provider": True,
|
||||
"exit_code": 1,
|
||||
}
|
||||
session = _session(args, ctx)
|
||||
payload: Dict[str, Any] = {"prompt": prompt}
|
||||
for key in ("model", "size", "quality"):
|
||||
if args.get(key):
|
||||
payload[key] = args[key]
|
||||
try:
|
||||
from src.ai_interaction import do_generate_image
|
||||
|
||||
data = await do_generate_image(
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
session_id=session,
|
||||
owner=owner,
|
||||
incognito=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
return "generate_image: error", {"error": f"generate_image failed: {exc}", "exit_code": 1}
|
||||
result = data.get("image_url") or data.get("image") or data.get("data")
|
||||
data_url = _as_data_url(result)
|
||||
if not data_url:
|
||||
return "generate_image", {"error": "generate_image returned no image.", "exit_code": 1}
|
||||
info: Dict[str, Any] = {
|
||||
"output": f"Generated an image for: {prompt[:120]}",
|
||||
"image_size": data.get("image_size"),
|
||||
"model": data.get("image_model") or str(args.get("model") or ""),
|
||||
"exit_code": 0,
|
||||
}
|
||||
try:
|
||||
info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), prompt, "generate_image"))
|
||||
except Exception as exc:
|
||||
logger.warning("generate_image persist failed: %s", exc)
|
||||
if bool(args.get("place", True)):
|
||||
placed = await _place_layer(
|
||||
data_url,
|
||||
str(args.get("name") or "Generated image"),
|
||||
owner,
|
||||
session,
|
||||
masked=bool(args.get("mask")),
|
||||
blend=str(args.get("blend") or "normal"),
|
||||
opacity=float(args.get("opacity") or 100),
|
||||
)
|
||||
info["placed"] = placed
|
||||
if placed:
|
||||
info["output"] += " Placed it as a new layer."
|
||||
return "generate_image", info
|
||||
@@ -0,0 +1,498 @@
|
||||
"""Editor agent/planner model resolution.
|
||||
|
||||
The image-editor Quick Edit planner (`/api/editor/plan`, `/api/editor/agent`)
|
||||
runs on a LOCAL Qwen3.5-9B vision model served on seneca when one is
|
||||
reachable, instead of the hosted chat model. This module owns:
|
||||
|
||||
* the local model base URLs / model id (settings-overridable),
|
||||
* an ordered candidate list (per-request ``agent_model`` override, then the
|
||||
local Qwen3.5-9B, then the hosted default/utility chat model),
|
||||
* a vision fallback chain: local model WITH the canvas screenshot, else a
|
||||
text-only call whose image is described by a separate VL caption model,
|
||||
* a lightweight reachability probe for the configured local bases.
|
||||
|
||||
Everything is best-effort: when no local server is reachable the planner
|
||||
transparently falls back to the previously-used hosted chat model, so the
|
||||
editor agent never regresses to "no plan".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The local Qwen3.5-9B vision server (seneca). :8097 (tmux AGENT_VLM) is the
|
||||
# canonical editor-agent endpoint. Override with the ``editor_agent_base_url``
|
||||
# setting (single URL wins) or the ``ODYSSEUS_EDITOR_AGENT_BASES`` env
|
||||
# (comma-separated). NOTE: :8095 is a DIFFERENT server (Qwen3-VL-30B) used ONLY
|
||||
# for the caption fallback below — never list it here as an agent model.
|
||||
DEFAULT_LOCAL_BASES = [
|
||||
"http://100.69.120.65:8097/v1",
|
||||
]
|
||||
DEFAULT_LOCAL_MODEL = "Qwen/Qwen3.5-9B"
|
||||
|
||||
# Vision caption model used when the editor-agent model cannot accept images.
|
||||
DEFAULT_CAPTION_BASE = "http://100.69.120.65:8095/v1"
|
||||
DEFAULT_CAPTION_MODEL = "Qwen/Qwen3.5-9B"
|
||||
|
||||
# Served-model-id cache per base (populated on first use). The local servers 400
|
||||
# with "Failed to load model" when asked for an id they do not serve (e.g. the
|
||||
# client's default "Qwen3.5-9B-MLX-8bit", or a stale caption model), so we
|
||||
# resolve the requested id against GET {base}/models and fall back to one that
|
||||
# is actually served.
|
||||
_MODELS_CACHE: Dict[str, set] = {}
|
||||
|
||||
_CAPTION_INSTRUCTION = (
|
||||
"Describe this image in one or two sentences for a photo editor: the main "
|
||||
"subject, the background/scene, lighting, dominant colours, and any obvious "
|
||||
"flaws (blown highlights, colour cast, clutter, blur)."
|
||||
)
|
||||
|
||||
|
||||
def _setting(key: str, default: Any = "") -> Any:
|
||||
try:
|
||||
from src.settings import load_settings
|
||||
|
||||
value = load_settings().get(key)
|
||||
return value if value not in (None, "") else default
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def local_bases() -> List[str]:
|
||||
override = _setting("editor_agent_base_url")
|
||||
if override:
|
||||
return [str(override)]
|
||||
env = os.getenv("ODYSSEUS_EDITOR_AGENT_BASES")
|
||||
if env:
|
||||
bases = [b.strip() for b in env.split(",") if b.strip()]
|
||||
if bases:
|
||||
return bases
|
||||
return list(DEFAULT_LOCAL_BASES)
|
||||
|
||||
|
||||
def local_model() -> str:
|
||||
return str(_setting("editor_agent_model", DEFAULT_LOCAL_MODEL))
|
||||
|
||||
|
||||
def caption_base() -> str:
|
||||
return str(_setting("editor_caption_base_url", DEFAULT_CAPTION_BASE))
|
||||
|
||||
|
||||
def caption_model() -> str:
|
||||
return str(_setting("editor_caption_model", DEFAULT_CAPTION_MODEL))
|
||||
|
||||
|
||||
# Per-request / per-plan context (contextvars keep this correct under the
|
||||
# async event loop without sharing state between concurrent editor turns).
|
||||
_override: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
|
||||
"editor_agent_model_override", default=None
|
||||
)
|
||||
_last_model: contextvars.ContextVar[Optional[Tuple[str, str]]] = contextvars.ContextVar(
|
||||
"editor_agent_last_model", default=None
|
||||
)
|
||||
|
||||
|
||||
def set_override(value: Optional[str]) -> None:
|
||||
_override.set(value or None)
|
||||
|
||||
|
||||
def get_override() -> Optional[str]:
|
||||
return _override.get()
|
||||
|
||||
|
||||
def set_last_model(model: str, source: str) -> None:
|
||||
_last_model.set((model, source))
|
||||
|
||||
|
||||
def get_last_model() -> Optional[Tuple[str, str]]:
|
||||
return _last_model.get()
|
||||
|
||||
|
||||
def _chat_url(base: str) -> str:
|
||||
b = str(base or "").rstrip("/")
|
||||
if b.endswith("/chat/completions"):
|
||||
return b
|
||||
return b + "/chat/completions"
|
||||
|
||||
|
||||
def _split_override(override: str) -> Tuple[str, str]:
|
||||
"""Parse ``base::model`` (or a bare model id)."""
|
||||
raw = str(override or "").strip()
|
||||
if "::" in raw:
|
||||
base, model = raw.split("::", 1)
|
||||
return base.strip(), model.strip()
|
||||
return "", raw
|
||||
|
||||
|
||||
def _has_image(messages: List[dict]) -> bool:
|
||||
for msg in messages:
|
||||
content = msg.get("content")
|
||||
if isinstance(content, list):
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get("type") == "image_url":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _first_image(messages: List[dict]) -> Optional[str]:
|
||||
for msg in messages:
|
||||
content = msg.get("content")
|
||||
if isinstance(content, list):
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get("type") == "image_url":
|
||||
url = (part.get("image_url") or {}).get("url")
|
||||
if isinstance(url, str):
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
def _strip_images(messages: List[dict]) -> List[dict]:
|
||||
out: List[dict] = []
|
||||
for msg in messages:
|
||||
content = msg.get("content")
|
||||
if isinstance(content, list):
|
||||
text = "\n".join(
|
||||
str(part.get("text") or "")
|
||||
for part in content
|
||||
if isinstance(part, dict) and part.get("type") == "text"
|
||||
)
|
||||
out.append({"role": msg.get("role", "user"), "content": text})
|
||||
else:
|
||||
out.append({"role": msg.get("role", "user"), "content": content})
|
||||
return out
|
||||
|
||||
|
||||
def _with_caption(messages: List[dict], caption: str) -> List[dict]:
|
||||
note = f"\n\n[Canvas screenshot caption: {caption}]"
|
||||
out = [dict(m) for m in messages]
|
||||
for msg in reversed(out):
|
||||
if msg.get("role") == "user":
|
||||
msg["content"] = str(msg.get("content") or "") + note
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
async def _post_chat(
|
||||
base: str,
|
||||
model: str,
|
||||
messages: List[dict],
|
||||
headers: Optional[dict],
|
||||
timeout: float,
|
||||
max_tokens: int = 1024,
|
||||
response_format: Optional[dict] = None,
|
||||
) -> Optional[str]:
|
||||
if not base or not model:
|
||||
return None
|
||||
payload: Dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": 0.2,
|
||||
"max_tokens": max_tokens,
|
||||
"stream": False,
|
||||
}
|
||||
# Constrain the planner to a strict JSON ops schema (structured outputs /
|
||||
# grammar) so the model can only emit valid editor ops. Unknown servers
|
||||
# ignore the field; the server-side schema validation is the real enforcer.
|
||||
if isinstance(response_format, dict):
|
||||
payload["response_format"] = response_format
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
resp = await client.post(_chat_url(base), json=payload, headers=headers or {})
|
||||
if resp.status_code != 200:
|
||||
logger.debug(
|
||||
"editor agent chat %s -> %s (%s)", base, resp.status_code, resp.text[:200]
|
||||
)
|
||||
return None
|
||||
data = resp.json()
|
||||
except Exception as exc:
|
||||
logger.debug("editor agent chat %s failed: %s", base, exc)
|
||||
return None
|
||||
try:
|
||||
return data["choices"][0]["message"]["content"]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def _served_models(base: str, timeout: float = 3.0) -> set:
|
||||
"""Return the set of model ids a base serves (cached; empty on failure)."""
|
||||
if not base:
|
||||
return set()
|
||||
cached = _MODELS_CACHE.get(base)
|
||||
if cached is not None:
|
||||
return cached
|
||||
ids: set = set()
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
resp = await client.get(base.rstrip("/") + "/models")
|
||||
if resp.status_code == 200:
|
||||
ids = {
|
||||
str(m.get("id"))
|
||||
for m in (resp.json().get("data") or [])
|
||||
if isinstance(m, dict) and m.get("id")
|
||||
}
|
||||
except Exception:
|
||||
ids = set()
|
||||
_MODELS_CACHE[base] = ids
|
||||
return ids
|
||||
|
||||
|
||||
async def _resolve_model(base: str, requested: str, fallback: str) -> str:
|
||||
"""Return a model id ``base`` actually serves.
|
||||
|
||||
If the requested id is served, keep it; otherwise use ``fallback`` (when
|
||||
served) or any served id. When the served list is unknown (server down),
|
||||
keep the requested id (or ``fallback``).
|
||||
"""
|
||||
requested = str(requested or "").strip()
|
||||
fallback = str(fallback or "").strip()
|
||||
ids = await _served_models(base)
|
||||
if ids:
|
||||
if requested and requested in ids:
|
||||
return requested
|
||||
if fallback in ids:
|
||||
return fallback
|
||||
return sorted(ids)[0]
|
||||
return requested or fallback
|
||||
|
||||
|
||||
async def caption_image(data_url: str, owner: Optional[str] = None, timeout: float = 90.0) -> Optional[str]:
|
||||
if not isinstance(data_url, str) or not data_url.startswith("data:image/"):
|
||||
return None
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": _CAPTION_INSTRUCTION},
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
],
|
||||
}
|
||||
]
|
||||
base = caption_base()
|
||||
model = await _resolve_model(base, caption_model(), local_model())
|
||||
return await _post_chat(base, model, messages, None, timeout, max_tokens=200)
|
||||
|
||||
|
||||
_DONE_RE = re.compile(r"^\s*DONE\b", re.I)
|
||||
_RETRY_RE = re.compile(r"RETRY:?\s*(.+)", re.I | re.S)
|
||||
|
||||
|
||||
def _parse_self_check(text: str) -> Dict[str, Any]:
|
||||
s = str(text or "").strip()
|
||||
if _DONE_RE.match(s):
|
||||
return {"done": True, "issues": [], "retry_hint": "", "text": s}
|
||||
m = _RETRY_RE.search(s)
|
||||
if m or re.search(r"\bretry\b", s, re.I):
|
||||
hint = (m.group(1).strip() if m else s) or s
|
||||
return {"done": False, "issues": [hint], "retry_hint": hint, "text": s}
|
||||
return {"done": True, "issues": [], "retry_hint": "", "text": s}
|
||||
|
||||
|
||||
async def self_check(
|
||||
before: Any,
|
||||
after: Any,
|
||||
prompt: str,
|
||||
model: Optional[str] = None,
|
||||
owner: Optional[str] = None,
|
||||
timeout: float = 30.0,
|
||||
) -> Dict[str, Any]:
|
||||
"""Same-origin VL self-check: compare BEFORE vs AFTER against the intent.
|
||||
|
||||
The browser cannot call the local VL server directly (it sends no CORS
|
||||
headers), so ``/api/editor/self-check`` proxies here. Returns
|
||||
``{done, issues, retry_hint, text}``; on any failure (bad input, the model
|
||||
is unreachable, or a timeout) it degrades to ``done=True`` with
|
||||
``reason="unavailable"`` so the client never gets stuck retrying.
|
||||
"""
|
||||
if not (
|
||||
isinstance(before, str) and before.startswith("data:image/")
|
||||
and isinstance(after, str) and after.startswith("data:image/")
|
||||
):
|
||||
return {"done": True, "issues": [], "retry_hint": "",
|
||||
"reason": "unavailable", "text": "DONE"}
|
||||
bases = local_bases() or list(DEFAULT_LOCAL_BASES)
|
||||
base = bases[0]
|
||||
mdl = await _resolve_model(base, str(model or local_model()), local_model())
|
||||
messages = [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": str(prompt or "")},
|
||||
{"type": "image_url", "image_url": {"url": before}},
|
||||
{"type": "image_url", "image_url": {"url": after}},
|
||||
],
|
||||
}]
|
||||
text = await _post_chat(base, mdl, messages, None, timeout, max_tokens=140)
|
||||
if not text:
|
||||
return {"done": True, "issues": [], "retry_hint": "",
|
||||
"reason": "unavailable", "text": "DONE"}
|
||||
out = _parse_self_check(text)
|
||||
out["model"] = mdl
|
||||
out["source"] = "local"
|
||||
return out
|
||||
|
||||
|
||||
def _candidates(owner: Optional[str], override: Optional[str]) -> List[Dict[str, Any]]:
|
||||
cands: List[Dict[str, Any]] = []
|
||||
if override:
|
||||
base, model = _split_override(override)
|
||||
if model:
|
||||
cands.append(
|
||||
{"base": base or local_bases()[0], "model": model, "headers": None, "source": "override"}
|
||||
)
|
||||
for base in local_bases():
|
||||
cands.append({"base": base, "model": local_model(), "headers": None, "source": "local"})
|
||||
try:
|
||||
from src.endpoint_resolver import resolve_endpoint
|
||||
|
||||
for prefix in ("utility", "default"):
|
||||
try:
|
||||
url, model, headers = resolve_endpoint(prefix, owner=owner)
|
||||
except Exception:
|
||||
continue
|
||||
if url and model:
|
||||
cands.append({"base": url, "model": model, "headers": headers, "source": "hosted"})
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return cands
|
||||
|
||||
|
||||
async def plan_with_editor_agent(
|
||||
messages: List[dict],
|
||||
owner: Optional[str] = None,
|
||||
override: Optional[str] = None,
|
||||
timeout: float = 90.0,
|
||||
response_format: Optional[dict] = None,
|
||||
) -> Optional[Tuple[str, str, str]]:
|
||||
"""Return (raw_plan_text, model_used, source) or None.
|
||||
|
||||
Tries the local model WITH the screenshot, then the same model text-only
|
||||
with a VL caption of the screenshot, then the hosted chat model.
|
||||
|
||||
``response_format`` (a JSON-schema structured-output constraint) is sent
|
||||
ONLY to the local editor-agent bases, so a hosted fallback model that does
|
||||
not support the field never rejects the request.
|
||||
"""
|
||||
has_img = _has_image(messages)
|
||||
text_messages = _strip_images(messages) if has_img else messages
|
||||
image = _first_image(messages) if has_img else None
|
||||
caption: Optional[str] = None
|
||||
_local = {str(b).rstrip("/") for b in local_bases()}
|
||||
for cand in _candidates(owner, override):
|
||||
base, model = cand["base"], cand["model"]
|
||||
headers = cand.get("headers")
|
||||
is_local = cand["source"] == "local" or str(base).rstrip("/") in _local
|
||||
rf = response_format if (response_format and is_local) else None
|
||||
extra = {"response_format": rf} if rf else {}
|
||||
if has_img:
|
||||
raw = await _post_chat(base, model, messages, headers, timeout, **extra)
|
||||
if raw:
|
||||
return raw, model, cand["source"]
|
||||
if caption is None and image:
|
||||
caption = await caption_image(image, owner)
|
||||
if caption:
|
||||
raw = await _post_chat(
|
||||
base, model, _with_caption(text_messages, caption), headers, timeout, **extra,
|
||||
)
|
||||
if raw:
|
||||
return raw, model, cand["source"] + "+caption"
|
||||
else:
|
||||
raw = await _post_chat(
|
||||
base, model, text_messages, headers, timeout, **extra
|
||||
)
|
||||
if raw:
|
||||
return raw, model, cand["source"]
|
||||
return None
|
||||
|
||||
|
||||
async def list_editor_agent_models(owner: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Models available to the editor agent, for the model picker.
|
||||
|
||||
Lists the local Qwen3.5-9B (one entry per reachable base) plus the hosted
|
||||
default/utility chat model that the planner falls back to.
|
||||
"""
|
||||
models: List[Dict[str, Any]] = []
|
||||
seen: set = set()
|
||||
for base in local_bases():
|
||||
model = local_model()
|
||||
key = ("local", base, model)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
models.append({
|
||||
"id": model,
|
||||
"label": f"{model} · local",
|
||||
"source": "local",
|
||||
"vision": True,
|
||||
"base_url": base,
|
||||
})
|
||||
try:
|
||||
from src.endpoint_resolver import resolve_endpoint
|
||||
|
||||
for prefix in ("utility", "default"):
|
||||
try:
|
||||
url, model, _headers = resolve_endpoint(prefix, owner=owner)
|
||||
except Exception:
|
||||
continue
|
||||
if url and model:
|
||||
key = ("hosted", model)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
models.append({
|
||||
"id": model,
|
||||
"label": model,
|
||||
"source": "hosted",
|
||||
"vision": None,
|
||||
"base_url": url,
|
||||
})
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return models
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reachability probe (best effort, non-blocking).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PROBE: Dict[str, bool] = {}
|
||||
|
||||
|
||||
def probe_models(base: str, timeout: float = 3.0) -> bool:
|
||||
try:
|
||||
with httpx.Client(timeout=timeout) as client:
|
||||
resp = client.get(str(base).rstrip("/") + "/models")
|
||||
return resp.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _probe_worker() -> None:
|
||||
for base in local_bases():
|
||||
ok = probe_models(base)
|
||||
_PROBE[base] = ok
|
||||
logger.info("editor agent local base %s reachable=%s", base, ok)
|
||||
cap = caption_base()
|
||||
_PROBE[cap] = probe_models(cap)
|
||||
logger.info("editor agent caption base %s reachable=%s", cap, _PROBE[cap])
|
||||
|
||||
|
||||
def startup_probe() -> None:
|
||||
"""Kick off a background reachability probe (never blocks startup)."""
|
||||
try:
|
||||
threading.Thread(target=_probe_worker, name="editor-agent-probe", daemon=True).start()
|
||||
except Exception:
|
||||
logger.debug("editor agent probe thread failed", exc_info=True)
|
||||
|
||||
|
||||
def probe_status() -> Dict[str, bool]:
|
||||
return dict(_PROBE)
|
||||
File diff suppressed because it is too large
Load Diff
+42
-23
@@ -68,6 +68,7 @@ def _row(index: int, message: dict[str, Any]) -> dict[str, Any]:
|
||||
})
|
||||
return {
|
||||
"uid": str(message.get("uid") or _uid(index)),
|
||||
"message_id": str(message.get("message_id") or ""),
|
||||
"account": str(message.get("account") or "fixture"),
|
||||
"account_email": str(message.get("account_email") or message.get("to") or ""),
|
||||
"account_id": str(message.get("account_id") or "fixture"),
|
||||
@@ -78,6 +79,16 @@ def _row(index: int, message: dict[str, Any]) -> dict[str, Any]:
|
||||
"folder": str(message.get("folder") or "INBOX"),
|
||||
"attachments": attachments,
|
||||
"has_attachments": bool(attachments),
|
||||
"summary": str(message.get("summary") or message.get("body") or ""),
|
||||
}
|
||||
|
||||
|
||||
def _account_matches(message: dict[str, Any], account: str | None) -> bool:
|
||||
selector = str(account or "").strip().casefold()
|
||||
return not selector or selector in {
|
||||
str(message.get("account") or "fixture").strip().casefold(),
|
||||
str(message.get("account_email") or message.get("to") or "").strip().casefold(),
|
||||
str(message.get("account_id") or "fixture").strip().casefold(),
|
||||
}
|
||||
|
||||
|
||||
@@ -108,9 +119,16 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
|
||||
|
||||
bare = tool.removeprefix("mcp__email__")
|
||||
if bare == "list_email_accounts":
|
||||
accounts = {}
|
||||
for index, message in enumerate(messages):
|
||||
row = _row(index, message)
|
||||
identity = (row["account_id"], row["account"], row["account_email"])
|
||||
accounts.setdefault(identity, {"id": row["account_id"], "name": row["account"],
|
||||
"email": row["account_email"], "default": not accounts})
|
||||
return {
|
||||
"accounts": [{"id": "fixture", "name": "Fixture mailbox", "default": True}],
|
||||
"output": "Fixture mailbox (default)",
|
||||
"accounts": list(accounts.values()),
|
||||
"output": "\n".join(f"{row['name']} <{row['email']}> ({row['id']})" for row in accounts.values())
|
||||
or "No fixture email accounts found.",
|
||||
"exit_code": 0,
|
||||
}
|
||||
|
||||
@@ -121,14 +139,7 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
|
||||
rows = [
|
||||
_row(i, m) for i, m in enumerate(messages)
|
||||
if _matches(m, query) and _folder_matches(m.get("folder"), folder)
|
||||
and (
|
||||
not account
|
||||
or account in {
|
||||
str(m.get("account") or "").strip().casefold(),
|
||||
str(m.get("account_email") or "").strip().casefold(),
|
||||
str(m.get("account_id") or "").strip().casefold(),
|
||||
}
|
||||
)
|
||||
and _account_matches(m, account)
|
||||
]
|
||||
limit = args.get("max_results", args.get("limit", 20))
|
||||
try:
|
||||
@@ -139,11 +150,7 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
|
||||
return {"output": "No emails found.", "emails": [], "exit_code": 0}
|
||||
output_lines = [f"Found {len(rows)} email(s):", ""]
|
||||
for index, r in enumerate(rows, start=1):
|
||||
source = next(
|
||||
(m for i, m in enumerate(messages) if _uid(i) == r["uid"]),
|
||||
{},
|
||||
)
|
||||
summary = str(source.get("summary") or source.get("body") or "").strip()
|
||||
summary = r["summary"].strip()
|
||||
output_lines.extend([
|
||||
f"{index}. **{r['subject']}**",
|
||||
f" From: {r['from']}",
|
||||
@@ -159,15 +166,19 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
|
||||
|
||||
if bare == "read_email":
|
||||
uid = str(args.get("uid") or "")
|
||||
message_id = str(args.get("message_id") or "")
|
||||
folder = str(args.get("folder") or "INBOX")
|
||||
try:
|
||||
index = int(uid) - 1
|
||||
except (TypeError, ValueError):
|
||||
index = -1
|
||||
if index < 0 or index >= len(messages) or not _folder_matches(messages[index].get("folder"), folder):
|
||||
return {"error": f"Email UID {uid} not found.", "exit_code": 1}
|
||||
message = messages[index]
|
||||
row = _row(index, message)
|
||||
if not uid and not message_id:
|
||||
return {"error": "read_email requires a UID or Message-ID.", "exit_code": 1}
|
||||
matches = [(m, _row(i, m)) for i, m in enumerate(messages)
|
||||
if _folder_matches(m.get("folder"), folder) and _account_matches(m, args.get("account"))
|
||||
and (not uid or str(m.get("uid") or _uid(i)) == uid)
|
||||
and (not message_id or str(m.get("message_id") or "") == message_id)]
|
||||
if not matches:
|
||||
return {"error": f"Email UID/Message-ID {uid or message_id} not found.", "exit_code": 1}
|
||||
if len(matches) != 1:
|
||||
return {"error": "Email identity is ambiguous; specify its account and folder.", "exit_code": 1}
|
||||
message, row = matches[0]
|
||||
output = (
|
||||
f"UID: {row['uid']}\nSubject: {row['subject']}\nFrom: {row['from']}\n"
|
||||
f"Date: {row['date']}\n\n{message.get('body') or ''}"
|
||||
@@ -185,6 +196,7 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
|
||||
all_messages = _messages()
|
||||
owner_value = str(owner or "").strip()
|
||||
visible_index = -1
|
||||
matches = []
|
||||
for original in all_messages:
|
||||
if owner_value and original.get("owner") and original.get("owner") != owner_value:
|
||||
continue
|
||||
@@ -193,6 +205,13 @@ def execute_fixture_email(tool: str, args: dict[str, Any], owner: str | None = N
|
||||
continue
|
||||
if not _folder_matches(original.get("folder"), folder):
|
||||
continue
|
||||
if not _account_matches(original, args.get("account")):
|
||||
continue
|
||||
matches.append(original)
|
||||
if len(matches) > 1:
|
||||
return {"error": "Email identity is ambiguous; specify its account and folder.", "exit_code": 1}
|
||||
if matches:
|
||||
original = matches[0]
|
||||
if bare == "archive_email":
|
||||
original["folder"] = "Archive"
|
||||
action = "Archived"
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Process-wide shutdown signalling for graceful restarts.
|
||||
|
||||
Server-Sent-Events / streaming responses never complete on their own, so
|
||||
uvicorn's graceful shutdown would wait for an open browser stream up to the
|
||||
full ``timeout_graceful_shutdown`` window before exiting. A SIGTERM/SIGINT
|
||||
handler flips this flag and tells uvicorn to exit; every streaming generator
|
||||
checks the flag on each loop and ends promptly, so a restart completes in
|
||||
seconds while ordinary in-flight requests still finish.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SHUTTING_DOWN = threading.Event()
|
||||
|
||||
|
||||
def is_shutting_down() -> bool:
|
||||
"""True once a shutdown (SIGTERM/SIGINT) has been requested."""
|
||||
return _SHUTTING_DOWN.is_set()
|
||||
|
||||
|
||||
def request_shutdown() -> None:
|
||||
"""Mark the process as shutting down (idempotent)."""
|
||||
if not _SHUTTING_DOWN.is_set():
|
||||
_SHUTTING_DOWN.set()
|
||||
logger.info("shutdown requested; streaming responses will end promptly")
|
||||
|
||||
|
||||
def reset() -> None:
|
||||
"""Clear the flag (for tests)."""
|
||||
_SHUTTING_DOWN.clear()
|
||||
+13
-3
@@ -16,7 +16,7 @@ from typing import Dict, Optional
|
||||
from src.tools._common import _parse_tool_args
|
||||
|
||||
|
||||
async def do_edit_image(content: str, owner: Optional[str] = None) -> Dict:
|
||||
async def do_edit_image(content: str, owner: Optional[str] = None, *, incognito: bool = False) -> Dict:
|
||||
"""Create an owner-scoped edited copy of a gallery image."""
|
||||
try:
|
||||
args = _parse_tool_args(content)
|
||||
@@ -45,7 +45,7 @@ async def do_edit_image(content: str, owner: Optional[str] = None) -> Dict:
|
||||
if not load_settings().get('image_gen_enabled', True):
|
||||
return {'error': 'Image generation is disabled by the administrator.', 'exit_code': 1}
|
||||
from src.ai_interaction import do_edit_image as edit_with_model
|
||||
return await edit_with_model(str(args.get('prompt') or '').strip(), info['path'], owner=owner, size='auto')
|
||||
return await edit_with_model(str(args.get('prompt') or '').strip(), info['path'], owner=owner, size='auto', incognito=incognito)
|
||||
|
||||
from core.database import GalleryImage, SessionLocal
|
||||
from src.constants import GENERATED_IMAGES_DIR
|
||||
@@ -79,7 +79,7 @@ async def do_edit_image(content: str, owner: Optional[str] = None) -> Dict:
|
||||
session_id = source.session_id
|
||||
db.close()
|
||||
from src.ai_interaction import do_edit_image as edit_with_model
|
||||
return await edit_with_model(prompt, str(source_path), session_id=session_id, owner=owner, size='auto')
|
||||
return await edit_with_model(prompt, str(source_path), session_id=session_id, owner=owner, size='auto', incognito=incognito)
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -111,6 +111,16 @@ async def do_edit_image(content: str, owner: Optional[str] = None) -> Dict:
|
||||
output_bytes = output.getvalue()
|
||||
width, height = image.size
|
||||
|
||||
if incognito:
|
||||
import base64
|
||||
return {
|
||||
'output': f'Image edited ({action}). Private result, not saved to gallery.',
|
||||
'exit_code': 0, 'image_id': None,
|
||||
'image_url': 'data:image/png;base64,' + base64.b64encode(output_bytes).decode('ascii'),
|
||||
'image_prompt': source.prompt or action, 'image_model': f'edit_image:{action}',
|
||||
'image_size': f'{width}x{height}', 'image_quality': source.quality or '',
|
||||
}
|
||||
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
filename = f"{uuid.uuid4().hex[:12]}.png"
|
||||
(root / filename).write_bytes(output_bytes)
|
||||
|
||||
+9
-70
@@ -63,80 +63,19 @@ INTERNAL_UPLOAD_URL_RE = re.compile(
|
||||
r"([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)"
|
||||
r"(?=$|[\s\"'<>\[\](){},;!?:&#]|\.(?![A-Za-z0-9]))"
|
||||
)
|
||||
# `<!--\s*pdf(?:_form)?_source\b[^>]*\bupload_id=["'](id)["'][^>]*-->` and
|
||||
# `\[Attachment:[^\]\r\n]*\|\s*id=(id)(?:\s*\||\s*\])` are matched by the
|
||||
# forward-only scanners below. As single regexes, every opener in a run with no
|
||||
# closing `>` / `]` rescanned to the end of that run: O(n^2) on chat content,
|
||||
# which is unbounded on persisted assistant output (CodeQL py/polynomial-redos).
|
||||
_PDF_SOURCE_OPEN_RE = re.compile(r"<!--\s*pdf(?:_form)?_source\b", re.IGNORECASE)
|
||||
_PDF_SOURCE_ID_RE = re.compile(
|
||||
r"\bupload_id=[\"']([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)[\"']",
|
||||
PDF_SOURCE_UPLOAD_RE = re.compile(
|
||||
r"<!--\s*pdf(?:_form)?_source\b[^>]*\bupload_id="
|
||||
r"[\"']([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)[\"'][^>]*-->",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_ATTACHMENT_REFERENCE_OPEN_RE = re.compile(r"\[Attachment:", re.IGNORECASE)
|
||||
_ATTACHMENT_REFERENCE_STOP_RE = re.compile(r"[\]\r\n]")
|
||||
_ATTACHMENT_REFERENCE_TAIL_RE = re.compile(
|
||||
r"\|\s*id=([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)(?:\s*\||\s*\])",
|
||||
ATTACHMENT_REFERENCE_LINE_RE = re.compile(
|
||||
r"\[Attachment:[^\]\r\n]*\|\s*id="
|
||||
r"([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)"
|
||||
r"(?:\s*\||\s*\])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _pdf_source_upload_ids(value: str) -> list[str]:
|
||||
"""IDs from `<!-- pdf_source ... upload_id="<id>" ... -->` comments.
|
||||
|
||||
Every opener before the next `>` shares that `>`: the comment matches only
|
||||
if `--` sits right before it, and then with the last `upload_id=` (the
|
||||
greedy `[^>]*` backtracks from the right). Otherwise all of those openers
|
||||
fail together, so the scan resumes after the `>`.
|
||||
"""
|
||||
found: list[str] = []
|
||||
pos = 0
|
||||
while True:
|
||||
opener = _PDF_SOURCE_OPEN_RE.search(value, pos)
|
||||
if opener is None:
|
||||
return found
|
||||
close = value.find(">", opener.end())
|
||||
if close < 0:
|
||||
return found
|
||||
if value[close - 2:close] == "--":
|
||||
last = None
|
||||
for last in _PDF_SOURCE_ID_RE.finditer(value, opener.end(), close):
|
||||
pass
|
||||
if last is not None:
|
||||
found.append(last.group(1))
|
||||
pos = close + 1
|
||||
|
||||
|
||||
def _attachment_reference_ids(value: str) -> list[str]:
|
||||
"""IDs from `[Attachment: name | id=<id> | ...]` reference lines.
|
||||
|
||||
The label scan stops at the first `]`/CR/LF, so every opener before that
|
||||
stop shares it and can only use the last `|` (scanning right to left)
|
||||
whose tail matches. If none does, all of those openers fail together and
|
||||
the scan resumes at the stop.
|
||||
"""
|
||||
found: list[str] = []
|
||||
pos = 0
|
||||
while True:
|
||||
opener = _ATTACHMENT_REFERENCE_OPEN_RE.search(value, pos)
|
||||
if opener is None:
|
||||
return found
|
||||
stop = _ATTACHMENT_REFERENCE_STOP_RE.search(value, opener.end())
|
||||
end = stop.start() if stop else len(value)
|
||||
tail = None
|
||||
pipe = value.rfind("|", opener.end(), end)
|
||||
while pipe >= 0:
|
||||
tail = _ATTACHMENT_REFERENCE_TAIL_RE.match(value, pipe)
|
||||
if tail is not None:
|
||||
break
|
||||
pipe = value.rfind("|", opener.end(), pipe)
|
||||
if tail is None:
|
||||
pos = end
|
||||
else:
|
||||
found.append(tail.group(1))
|
||||
pos = tail.end()
|
||||
|
||||
|
||||
def is_valid_upload_id(upload_id: str) -> bool:
|
||||
"""Return True when *upload_id* matches the canonical uploads.json id format."""
|
||||
return UPLOAD_ID_RE.fullmatch(upload_id or "") is not None
|
||||
@@ -171,8 +110,8 @@ def extract_internal_upload_ids(value: Any) -> set[str]:
|
||||
return set()
|
||||
return (
|
||||
set(INTERNAL_UPLOAD_URL_RE.findall(value))
|
||||
| set(_pdf_source_upload_ids(value))
|
||||
| set(_attachment_reference_ids(value))
|
||||
| set(PDF_SOURCE_UPLOAD_RE.findall(value))
|
||||
| set(ATTACHMENT_REFERENCE_LINE_RE.findall(value))
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user