diff --git a/.claude/skills/glb-gen/SKILL.md b/.claude/skills/glb-gen/SKILL.md new file mode 100644 index 000000000..0f9f08071 --- /dev/null +++ b/.claude/skills/glb-gen/SKILL.md @@ -0,0 +1,96 @@ +--- +name: glb-gen +description: > + Convert concept images to game-ready .glb 3D models for The Settled Reach + using Trellis (image-to-3D on tower-of-joy:11510) and Blender (post-processing). + Input is a PNG image. Output is a .glb file. Use when the user says "convert + to 3d", "make glb", "trellis", "image to 3d", "glb-gen", or has approved + concept images ready for 3D conversion. NOT for generating concept images — + use /image-gen for that. +--- + +# GLB Generation — Image to 3D Model + +Convert approved concept images to game-ready .glb models. + +## Prerequisites + +| Service | Check | +|---------|-------| +| Trellis | `tooling/db/trellis_connector.py health` | +| Blender | `flatpak run org.blender.Blender --version` | + +Trellis runs on tower-of-joy and may be switched off. Check before batching. + +## Usage + +```bash +python3 tooling/db/trellis_connector.py generate input.png \ + --output .tmp/glb-gen/[name].glb \ + --simplify 0.95 \ + --texture-size 1024 +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--output` | auto-named | Output .glb path | +| `--simplify` | 0.95 | Mesh simplification (0.9=aggressive, 0.98=gentle) | +| `--texture-size` | 1024 | Baked texture resolution (512-2048) | +| `--seed` | 0 | Random seed for reproducibility | +| `--timeout` | 600 | Max wait seconds | + +## Intermediate and Output Directories + +All intermediates go to `.tmp/` in the project root (gitignored, findable). +Use deep nesting by pipeline stage, asset category, and subcategory: + +``` +.tmp/ + image-gen/ ← concept images (from /image-gen) + furniture/tables/baroque_table_concept.png + characters/body/slim_body_concept.png + glb-gen/ ← raw Trellis output + furniture/tables/baroque_table.glb + characters/body/slim_body.glb + glb-gen/postproc/ ← Blender post-processed + furniture/tables/baroque_table.glb + characters/body/slim_body.glb +``` + +Always mirror the category/subcategory path across stages so you can trace +`image-gen/furniture/tables/foo_concept.png` → `glb-gen/furniture/tables/foo.glb`. + +Final game-ready assets are copied to `client-tmp/models/[category]/` for +spike testing, or to `client/assets/models/` when ready for production. + +## Post-process in Blender (optional) + +Reassign materials to game standard names, adjust scale: + +```bash +flatpak run org.blender.Blender --background \ + --python .claude/skills/glb-gen/scripts/postprocess_glb.py \ + -- input.glb output.glb [--scale FACTOR] +``` + +## Input Requirements + +For best Trellis results: +- Square image (1:1), PNG format +- Plain dark background, single object centered +- Near-white/light base color +- No text, labels, or watermarks + +These match `/image-gen` output with the Settled Reach style guide. + +## Batch Usage + +Each `trellis_connector.py generate` call is independent. Run sequentially +(Trellis uses GPU — one job at a time) or queue them. Each call is non-blocking +relative to other skills. + +## Material Convention + +Post-processed .glb files use standard material slot names: +`mat_wood_primary`, `mat_metal_primary`, `mat_fabric_primary`, etc. +See `docs/design/character-visuals-spec.md` §5 for character materials. diff --git a/.claude/skills/glb-gen/scripts/postprocess b/.claude/skills/glb-gen/scripts/postprocess new file mode 100755 index 000000000..a6d48929a --- /dev/null +++ b/.claude/skills/glb-gen/scripts/postprocess @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Wrapper for the Blender GLB post-processor. +# +# Usage: +# postprocess [options] +# postprocess .tmp/glb-gen/furniture/desks/scifi_desk.glb client-tmp/models/furniture/scifi_desk.glb +# postprocess .tmp/glb-gen/furniture/desks/scifi_desk.glb # auto-output to .tmp/glb-gen/postproc/... +# +# Options are passed through to the Blender script: +# --target-width FLOAT Target width in world units (default: 1.0) +# --target-height FLOAT Target height in world units (default: auto) +# --color-threshold FLOAT Dominant color detection threshold (default: 0.25) +# --material-name NAME Material slot name (default: mat_primary) +# +# Output: +# Post-processed model (texture preserved) +# Recolor mask sidecar (if texture found) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BLENDER_SCRIPT="$SCRIPT_DIR/postprocess_glb.py" + +# Find project root via git +PROJECT_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" + +if [ $# -lt 1 ]; then + echo "Usage: postprocess [output.glb] [--target-width N] [--color-threshold N]" + echo "" + echo "If output is omitted, writes to .tmp/glb-gen/postproc/ mirroring the input path." + exit 1 +fi + +INPUT_REL="$1" +shift + +# Resolve to absolute +if [[ "$INPUT_REL" = /* ]]; then + INPUT_ABS="$INPUT_REL" +else + INPUT_ABS="$(cd "$PROJECT_ROOT" && pwd)/$INPUT_REL" +fi + +if [ ! -f "$INPUT_ABS" ]; then + echo "ERROR: Input file not found: $INPUT_ABS" >&2 + exit 1 +fi + +# Determine output path +if [ $# -ge 1 ] && [[ "$1" != --* ]]; then + OUTPUT_REL="$1" + shift + if [[ "$OUTPUT_REL" = /* ]]; then + OUTPUT_ABS="$OUTPUT_REL" + else + OUTPUT_ABS="$(cd "$PROJECT_ROOT" && pwd)/$OUTPUT_REL" + fi +else + # Auto-output: mirror input path under .tmp/glb-gen/postproc/ + # e.g. .tmp/glb-gen/furniture/desks/foo.glb → .tmp/glb-gen/postproc/furniture/desks/foo.glb + REL_TO_TMP="${INPUT_REL#.tmp/glb-gen/}" + OUTPUT_ABS="$(cd "$PROJECT_ROOT" && pwd)/.tmp/glb-gen/postproc/$REL_TO_TMP" +fi + +mkdir -p "$(dirname "$OUTPUT_ABS")" + +echo "Post-processing: $(basename "$INPUT_ABS")" +echo " Input: $INPUT_REL" +echo " Output: ${OUTPUT_ABS#$(cd "$PROJECT_ROOT" && pwd)/}" + +flatpak run org.blender.Blender --background \ + --python "$BLENDER_SCRIPT" \ + -- "$INPUT_ABS" "$OUTPUT_ABS" "$@" 2>&1 \ + | grep -E "^ |^=|WARNING|ERROR|Dominant|Mask" + +MASK="${OUTPUT_ABS%.glb}_mask.png" +if [ -f "$MASK" ]; then + echo " Mask: ${MASK#$(cd "$PROJECT_ROOT" && pwd)/}" +fi + +echo "Done." diff --git a/.claude/skills/glb-gen/scripts/postprocess_glb.py b/.claude/skills/glb-gen/scripts/postprocess_glb.py new file mode 100644 index 000000000..d21f8f16f --- /dev/null +++ b/.claude/skills/glb-gen/scripts/postprocess_glb.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +""" +GLB post-processor for The Settled Reach asset pipeline. + +Run via Blender headless: + flatpak run org.blender.Blender --background --python postprocess_glb.py -- input.glb output.glb [options] + +Operations: + 1. Normalize scale to fit a target bounding box (default 1x1x1 world units) + 2. Preserve Trellis texture, bake a recolor mask for the dominant color region + 3. Center the model on the origin, feet on the floor (Y=0) + 4. Export clean .glb + mask PNG sidecar + +The mask texture marks the dominant color region (white = replaceable by engine +tint, black = keep original Trellis detail). Godot loads the mask as a second +texture and uses a color-key shader for runtime recoloring. + +Options: + --target-width FLOAT Target width in world units (default: 1.0) + --target-height FLOAT Target height in world units (default: auto from aspect ratio) + --color-threshold FLOAT Distance threshold for dominant color detection (default: 0.25) + --material-name NAME Material slot name (default: mat_primary) +""" + +import bpy +import bmesh +import sys +import os +import math +from mathutils import Vector + + +def get_script_args(): + """Extract arguments after '--' from Blender's sys.argv.""" + try: + idx = sys.argv.index("--") + return sys.argv[idx + 1:] + except ValueError: + return [] + + +def parse_args(args): + """Parse script arguments.""" + if len(args) < 2: + print("Usage: postprocess_glb.py -- input.glb output.glb [--target-width N] [--color #hex] [--material-name name]") + sys.exit(1) + + result = { + "input": args[0], + "output": args[1], + "target_width": 1.0, + "target_height": None, + "color_threshold": 0.25, + "material_name": "mat_primary", + } + + i = 2 + while i < len(args): + if args[i] == "--target-width" and i + 1 < len(args): + result["target_width"] = float(args[i + 1]) + i += 2 + elif args[i] == "--target-height" and i + 1 < len(args): + result["target_height"] = float(args[i + 1]) + i += 2 + elif args[i] == "--color-threshold" and i + 1 < len(args): + result["color_threshold"] = float(args[i + 1]) + i += 2 + elif args[i] == "--material-name" and i + 1 < len(args): + result["material_name"] = args[i + 1] + i += 2 + else: + print(f"Unknown argument: {args[i]}") + i += 1 + + return result + + +def clear_scene(): + """Remove all objects from the scene.""" + bpy.ops.object.select_all(action='SELECT') + bpy.ops.object.delete() + # Clear orphan data + for block in bpy.data.meshes: + if block.users == 0: + bpy.data.meshes.remove(block) + for block in bpy.data.materials: + if block.users == 0: + bpy.data.materials.remove(block) + for block in bpy.data.images: + if block.users == 0: + bpy.data.images.remove(block) + + +def import_glb(filepath): + """Import a .glb file.""" + print(f" Importing {filepath}...") + bpy.ops.import_scene.gltf(filepath=filepath) + return [obj for obj in bpy.context.scene.objects if obj.type == 'MESH'] + + +def get_combined_bbox(objects): + """Get the combined bounding box of all mesh objects.""" + min_co = Vector((float('inf'), float('inf'), float('inf'))) + max_co = Vector((float('-inf'), float('-inf'), float('-inf'))) + + for obj in objects: + for corner in obj.bound_box: + world_co = obj.matrix_world @ Vector(corner) + min_co.x = min(min_co.x, world_co.x) + min_co.y = min(min_co.y, world_co.y) + min_co.z = min(min_co.z, world_co.z) + max_co.x = max(max_co.x, world_co.x) + max_co.y = max(max_co.y, world_co.y) + max_co.z = max(max_co.z, world_co.z) + + return min_co, max_co + + +def normalize_scale(objects, target_width, target_height=None): + """Scale all objects so the combined bounding box fits the target size.""" + min_co, max_co = get_combined_bbox(objects) + size = max_co - min_co + + if size.x == 0 and size.y == 0 and size.z == 0: + print(" WARNING: Zero-size bounding box, skipping scale normalization") + return 1.0 + + # In Blender: X=right, Y=forward, Z=up + # Our game: width is max(X, Y), height is Z + current_width = max(size.x, size.y) + current_height = size.z + + if current_width == 0: + current_width = 0.001 + + # Scale to target width + scale_factor = target_width / current_width + + # If target height specified, use the more constraining dimension + if target_height is not None and current_height > 0: + height_scale = target_height / current_height + scale_factor = min(scale_factor, height_scale) + + print(f" Current size: {size.x:.3f} x {size.y:.3f} x {size.z:.3f}") + print(f" Scale factor: {scale_factor:.4f}") + print(f" Result size: {size.x * scale_factor:.3f} x {size.y * scale_factor:.3f} x {size.z * scale_factor:.3f}") + + # Apply scale to all objects + for obj in objects: + obj.scale *= scale_factor + + # Apply transforms + bpy.ops.object.select_all(action='SELECT') + bpy.ops.object.transform_apply(location=False, rotation=False, scale=True) + + return scale_factor + + +def center_on_origin(objects): + """Center the combined bounding box on origin, bottom at Z=0.""" + min_co, max_co = get_combined_bbox(objects) + center = (min_co + max_co) / 2.0 + + # Move so center X/Y is at origin, bottom Z is at 0 + offset = Vector((-center.x, -center.y, -min_co.z)) + + for obj in objects: + obj.location += offset + + # Apply location + bpy.ops.object.select_all(action='SELECT') + bpy.ops.object.transform_apply(location=True, rotation=False, scale=False) + + print(f" Centered: offset applied ({offset.x:.3f}, {offset.y:.3f}, {offset.z:.3f})") + + +def find_texture_image(objects): + """Find the base color texture image from the imported materials.""" + for obj in objects: + if obj.type != 'MESH': + continue + for slot in obj.material_slots: + mat = slot.material + if mat is None or not mat.use_nodes: + continue + for node in mat.node_tree.nodes: + if node.type == 'TEX_IMAGE' and node.image is not None: + return node.image + return None + + +def analyze_dominant_color(image): + """Find the dominant color in a Blender image by pixel frequency. + + Quantizes to 8-bit buckets (32 levels per channel) and returns the + center of the largest bucket as (r, g, b). + """ + pixels = list(image.pixels) # flat RGBA + width, height = image.size + total = width * height + + # Quantize into buckets (5-bit per channel = 32 levels) + LEVELS = 32 + buckets = {} + for i in range(0, len(pixels), 4): + r, g, b = pixels[i], pixels[i + 1], pixels[i + 2] + a = pixels[i + 3] + if a < 0.1: + continue # skip transparent + qr = int(r * (LEVELS - 1)) + qg = int(g * (LEVELS - 1)) + qb = int(b * (LEVELS - 1)) + key = (qr, qg, qb) + if key in buckets: + buckets[key][0] += 1 + buckets[key][1] += r + buckets[key][2] += g + buckets[key][3] += b + else: + buckets[key] = [1, r, g, b] + + if not buckets: + return (0.8, 0.8, 0.8) + + # Find the largest bucket + best_key = max(buckets, key=lambda k: buckets[k][0]) + count, sum_r, sum_g, sum_b = buckets[best_key] + dominant = (sum_r / count, sum_g / count, sum_b / count) + + pct = (count / total) * 100 if total > 0 else 0 + print(f" Dominant color: ({dominant[0]:.2f}, {dominant[1]:.2f}, {dominant[2]:.2f}) — {pct:.1f}% of pixels") + + return dominant + + +def generate_mask(image, dominant_color, threshold, output_path): + """Generate a recolor mask: white where pixels are near the dominant color, + black elsewhere. Saves as a PNG sidecar next to the GLB.""" + pixels = list(image.pixels) + width, height = image.size + dr, dg, db = dominant_color + + mask_pixels = [0.0] * (width * height * 4) + replaceable_count = 0 + total_count = 0 + + for i in range(0, len(pixels), 4): + px_idx = i // 4 + r, g, b, a = pixels[i], pixels[i + 1], pixels[i + 2], pixels[i + 3] + + if a < 0.1: + # Transparent — not replaceable + mask_pixels[i + 3] = 0.0 + continue + + total_count += 1 + # Euclidean distance in RGB space + dist = ((r - dr) ** 2 + (g - dg) ** 2 + (b - db) ** 2) ** 0.5 + + if dist <= threshold: + # Replaceable region — white + mask_pixels[i] = 1.0 + mask_pixels[i + 1] = 1.0 + mask_pixels[i + 2] = 1.0 + mask_pixels[i + 3] = 1.0 + replaceable_count += 1 + else: + # Detail region — black + mask_pixels[i] = 0.0 + mask_pixels[i + 1] = 0.0 + mask_pixels[i + 2] = 0.0 + mask_pixels[i + 3] = 1.0 + + pct = (replaceable_count / total_count * 100) if total_count > 0 else 0 + print(f" Mask: {replaceable_count}/{total_count} pixels replaceable ({pct:.1f}%)") + + # Create a new Blender image for the mask + mask_img = bpy.data.images.new("recolor_mask", width, height, alpha=True) + mask_img.pixels = mask_pixels + mask_img.filepath_raw = output_path + mask_img.file_format = 'PNG' + mask_img.save() + print(f" Mask saved: {output_path}") + + return mask_img + + +def setup_materials(objects, material_name): + """Keep the original Trellis texture but rename the material slot. + Sets roughness high and specular low for toon compatibility.""" + for obj in objects: + if obj.type != 'MESH': + continue + for slot in obj.material_slots: + mat = slot.material + if mat is None: + continue + mat.name = material_name + if not mat.use_nodes: + mat.roughness = 1.0 + mat.specular_intensity = 0.0 + else: + # Find the Principled BSDF and adjust for toon + for node in mat.node_tree.nodes: + if node.type == 'BSDF_PRINCIPLED': + node.inputs['Roughness'].default_value = 1.0 + node.inputs['Specular IOR Level'].default_value = 0.0 + print(f" Material renamed to '{material_name}', roughness=1.0, specular=0.0") + + +def export_glb(filepath): + """Export the scene as .glb, preserving embedded textures.""" + print(f" Exporting to {filepath}...") + os.makedirs(os.path.dirname(os.path.abspath(filepath)), exist_ok=True) + + # Ensure all images are packed — Trellis GLBs embed textures, but after + # Blender import they may become external references that point nowhere. + packed = 0 + for img in bpy.data.images: + if img.packed_file is None and img.filepath: + try: + img.pack() + packed += 1 + except Exception as e: + print(f" WARNING: Could not pack image '{img.name}': {e}") + if packed: + print(f" Packed {packed} image(s) back into the blend data") + + bpy.ops.export_scene.gltf( + filepath=filepath, + export_format='GLB', + use_selection=False, + export_apply=True, + export_materials='EXPORT', + export_image_format='PNG', + ) + + size = os.path.getsize(filepath) + print(f" Exported: {filepath} ({size} bytes)") + + +def main(): + args = parse_args(get_script_args()) + + print(f"\n=== GLB Post-Processor ===") + print(f" Input: {args['input']}") + print(f" Output: {args['output']}") + print(f" Target width: {args['target_width']}") + print(f" Color threshold: {args['color_threshold']}") + print(f" Material: {args['material_name']}") + print() + + # Clear and import + clear_scene() + mesh_objects = import_glb(args['input']) + + if not mesh_objects: + print(" ERROR: No mesh objects found in .glb") + sys.exit(1) + + print(f" Found {len(mesh_objects)} mesh object(s)") + + # Normalize scale + normalize_scale(mesh_objects, args['target_width'], args['target_height']) + + # Center on origin, feet on floor + center_on_origin(mesh_objects) + + # Analyze texture and generate recolor mask (if texture exists) + tex_image = find_texture_image(mesh_objects) + if tex_image: + dominant = analyze_dominant_color(tex_image) + mask_path = os.path.splitext(os.path.abspath(args['output']))[0] + "_mask.png" + generate_mask(tex_image, dominant, args['color_threshold'], mask_path) + # Keep original texture, just rename material and adjust PBR + setup_materials(mesh_objects, args['material_name']) + else: + print(" No texture found — Trellis output has no atlas.") + print(" Keeping existing material as-is (flat color, single surface).") + setup_materials(mesh_objects, args['material_name']) + + # Export (preserves texture in GLB) + export_glb(args['output']) + + print("\n=== Done ===\n") + + +if __name__ == "__main__": + main() diff --git a/tooling/db/config.json b/tooling/db/config.json index 1b69b0a7b..6d0eab87e 100644 --- a/tooling/db/config.json +++ b/tooling/db/config.json @@ -2,6 +2,7 @@ "qdrant_url": "http://tower-of-joy:6333", "ollama_url": "http://tower-of-joy:11434", "stable_audio_url": "http://tower-of-joy:11500", + "trellis_url": "http://tower-of-joy:11510", "collection": "commonwealth", "embed_model": "nomic-embed-text", "embed_dimensions": 768 diff --git a/tooling/db/image_connector.py b/tooling/db/image_connector.py new file mode 100755 index 000000000..6c0884991 --- /dev/null +++ b/tooling/db/image_connector.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +""" +Gemini image generator connector — direct API wrapper. + +Generates images via Google's Gemini 2.0 Flash image generation API. +API key from GEMINI_API_KEY env var or config.json. + +Usage: + python3 image_connector.py health + python3 image_connector.py generate "prompt" [--output file.png] [--aspect 1:1] [--size 1K] [--input image.png] +""" + +import base64 +import json +import os +import sys +import urllib.error +import urllib.request + +CONFIG_PATH = os.path.join(os.path.dirname(__file__), "config.json") +DEFAULT_OUTPUT_DIR = os.path.expanduser("~/Pictures/mcp-images") + + +def get_api_key(): + """Get Gemini API key from env or config.""" + key = os.environ.get("GEMINI_API_KEY") + if key: + return key + try: + with open(CONFIG_PATH) as f: + config = json.load(f) + return config.get("gemini_api_key", "") + except Exception: + pass + print(json.dumps({ + "ok": False, + "error": "No GEMINI_API_KEY found in environment or config.json" + }, indent=2)) + sys.exit(1) + + +def health(): + """Check if the Gemini API is reachable with the configured key.""" + key = get_api_key() + url = f"https://generativelanguage.googleapis.com/v1beta/models?key={key}" + try: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read()) + models = [m.get("name", "") for m in data.get("models", []) + if "imagen" in m.get("name", "").lower() + or "flash" in m.get("name", "").lower()] + print(json.dumps({ + "ok": True, + "api": "gemini", + "image_capable_models": models[:5], + }, indent=2)) + except Exception as e: + print(json.dumps({ + "ok": False, + "error": str(e) + }, indent=2)) + sys.exit(1) + + +def generate(prompt, output=None, aspect_ratio="1:1", image_size=None, + input_image=None): + """ + Generate an image from a text prompt using Gemini. + + Args: + prompt: Text description of the image to generate + output: Output file path (default: auto-named in ~/Pictures/mcp-images/) + aspect_ratio: Aspect ratio (1:1, 16:9, 3:2, etc.) + image_size: Resolution hint (1K, 2K, 4K) - may not be honored + input_image: Optional input image path for image-to-image generation + """ + key = get_api_key() + + # Gemini image generation model + model = "gemini-2.5-flash-image" + url = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={key}" + + if output is None: + safe = "".join(c if c.isalnum() or c in "-_ " else "" for c in prompt[:40]) + safe = safe.strip().replace(" ", "_").lower() + os.makedirs(DEFAULT_OUTPUT_DIR, exist_ok=True) + output = os.path.join(DEFAULT_OUTPUT_DIR, f"{safe}.png") + + # Build the request + parts = [] + + # Add input image if provided (image-to-image) + if input_image: + if not os.path.isfile(input_image): + print(json.dumps({"ok": False, "error": f"Input image not found: {input_image}"}), indent=2) + sys.exit(1) + with open(input_image, "rb") as f: + image_data = base64.b64encode(f.read()).decode("utf-8") + # Detect mime type + ext = os.path.splitext(input_image)[1].lower() + mime = {"png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", + ".webp": "image/webp"}.get(ext, "image/png") + parts.append({ + "inlineData": { + "mimeType": mime, + "data": image_data + } + }) + + # Build enhanced prompt with aspect ratio and size hints + enhanced_prompt = prompt + if aspect_ratio and aspect_ratio != "1:1": + enhanced_prompt += f" Aspect ratio: {aspect_ratio}." + if image_size: + enhanced_prompt += f" Resolution: {image_size}." + + parts.append({"text": enhanced_prompt}) + + payload = json.dumps({ + "contents": [{"parts": parts}], + "generationConfig": { + "responseModalities": ["TEXT", "IMAGE"], + } + }) + + req = urllib.request.Request( + url, + data=payload.encode(), + headers={"Content-Type": "application/json"}, + method="POST" + ) + + print(f"Generating image...", file=sys.stderr) + print(f" Prompt: {prompt}", file=sys.stderr) + if input_image: + print(f" Input image: {input_image}", file=sys.stderr) + + try: + with urllib.request.urlopen(req, timeout=120) as resp: + result = json.loads(resp.read()) + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + print(json.dumps({ + "ok": False, + "error": f"API error {e.code}: {e.reason}", + "details": body[:500] + }, indent=2)) + sys.exit(1) + except Exception as e: + print(json.dumps({"ok": False, "error": str(e)}), indent=2) + sys.exit(1) + + # Extract image data from response + candidates = result.get("candidates", []) + if not candidates: + print(json.dumps({ + "ok": False, + "error": "No candidates in response", + "response": json.dumps(result)[:500] + }, indent=2)) + sys.exit(1) + + image_saved = False + text_response = "" + + for candidate in candidates: + content = candidate.get("content", {}) + for part in content.get("parts", []): + if "inlineData" in part: + # Image data + image_b64 = part["inlineData"]["data"] + image_bytes = base64.b64decode(image_b64) + os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True) + with open(output, "wb") as f: + f.write(image_bytes) + image_saved = True + elif "text" in part: + text_response += part["text"] + + if not image_saved: + print(json.dumps({ + "ok": False, + "error": "No image data in response", + "text_response": text_response[:500], + "response": json.dumps(result)[:500] + }, indent=2)) + sys.exit(1) + + file_size = os.path.getsize(output) + print(json.dumps({ + "ok": True, + "file": output, + "size_bytes": file_size, + "prompt": prompt, + "aspect_ratio": aspect_ratio, + }, indent=2)) + + +def main(): + if len(sys.argv) < 2: + print("Usage:") + print(" image_connector.py health") + print(" image_connector.py generate 'prompt' [--output file.png] [--aspect 1:1] [--size 1K] [--input image.png]") + sys.exit(1) + + cmd = sys.argv[1] + + if cmd == "health": + health() + elif cmd == "generate": + if len(sys.argv) < 3: + print("Error: prompt required", file=sys.stderr) + sys.exit(1) + + prompt = sys.argv[2] + output = None + aspect_ratio = "1:1" + image_size = None + input_image = None + + i = 3 + while i < len(sys.argv): + if sys.argv[i] == "--output" and i + 1 < len(sys.argv): + output = sys.argv[i + 1] + i += 2 + elif sys.argv[i] == "--aspect" and i + 1 < len(sys.argv): + aspect_ratio = sys.argv[i + 1] + i += 2 + elif sys.argv[i] == "--size" and i + 1 < len(sys.argv): + image_size = sys.argv[i + 1] + i += 2 + elif sys.argv[i] == "--input" and i + 1 < len(sys.argv): + input_image = sys.argv[i + 1] + i += 2 + else: + print(f"Unknown argument: {sys.argv[i]}", file=sys.stderr) + sys.exit(1) + + generate(prompt, output=output, aspect_ratio=aspect_ratio, + image_size=image_size, input_image=input_image) + else: + print(f"Unknown command: {cmd}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tooling/db/trellis_connector.py b/tooling/db/trellis_connector.py new file mode 100755 index 000000000..3d28b3831 --- /dev/null +++ b/tooling/db/trellis_connector.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +""" +Trellis 3D model generator connector — Gradio API wrapper. + +Talks to the Trellis Gradio app at tower-of-joy:11510. +Pipeline: upload image → start session → image_to_3d → extract_glb → download .glb + +Usage: + python3 trellis_connector.py health + python3 trellis_connector.py generate image.png [--output model.glb] [--simplify 0.95] [--texture-size 1024] [--seed 42] [--timeout 600] +""" + +import base64 +import json +import os +import shutil +import sys +import time +import urllib.error +import urllib.request +import urllib.parse + +CONFIG_PATH = os.path.join(os.path.dirname(__file__), "config.json") + + +def load_config(): + with open(CONFIG_PATH) as f: + return json.load(f) + + +def get_base_url(): + config = load_config() + return config.get("trellis_url", "http://tower-of-joy:11510") + + +def health(): + """Check if the Trellis API is reachable.""" + base = get_base_url() + try: + req = urllib.request.Request(f"{base}/info", method="GET") + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read()) + endpoints = list(data.get("named_endpoints", {}).keys()) + print(json.dumps({ + "ok": True, + "url": base, + "endpoints": endpoints + }, indent=2)) + except Exception as e: + print(json.dumps({ + "ok": False, + "url": base, + "error": str(e) + }, indent=2)) + sys.exit(1) + + +def _call_api(base, endpoint, data, timeout=600): + """Call a Gradio API endpoint. Tries sync /api/ first, falls back to SSE /gradio_api/call/.""" + # Trellis uses the sync /api/ pattern + api_url = f"{base}/api{endpoint}" + + payload = json.dumps({"data": data}) + req = urllib.request.Request( + api_url, + data=payload.encode(), + headers={"Content-Type": "application/json"}, + method="POST" + ) + + print(f" Calling {endpoint}...", file=sys.stderr) + + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + result = json.loads(resp.read()) + # Sync Gradio returns {"data": [...], "is_generating": false, ...} + if isinstance(result, dict) and "data" in result: + return result["data"] + return result + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"{endpoint} failed ({e.code}): {body[:300]}") + + +def _upload_image(base, image_path): + """Upload an image file to the Gradio server and return the file reference.""" + upload_url = f"{base}/upload" + + with open(image_path, "rb") as f: + image_data = f.read() + + filename = os.path.basename(image_path) + + # Gradio upload expects multipart/form-data with a 'files' field + boundary = "----TrellisConnectorBoundary" + body = ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="files"; filename="{filename}"\r\n' + f"Content-Type: image/png\r\n" + f"\r\n" + ).encode() + image_data + f"\r\n--{boundary}--\r\n".encode() + + req = urllib.request.Request( + upload_url, + data=body, + headers={ + "Content-Type": f"multipart/form-data; boundary={boundary}", + }, + method="POST" + ) + + print(f" Uploading {filename}...", file=sys.stderr) + with urllib.request.urlopen(req, timeout=30) as resp: + result = json.loads(resp.read()) + # Gradio returns a list of uploaded file paths + if isinstance(result, list) and len(result) > 0: + return result[0] + raise RuntimeError(f"Upload failed: {result}") + + +def _download_file(url, output_path, base): + """Download a file from the Gradio server.""" + if url.startswith("/"): + url = f"{base}{url}" + elif not url.startswith("http"): + url = f"{base}/file={url}" + + print(f" Downloading to {output_path}...", file=sys.stderr) + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=120) as resp: + with open(output_path, "wb") as f: + shutil.copyfileobj(resp, f) + + return os.path.getsize(output_path) + + +def _check_available(base): + """Quick check if Trellis is reachable. Fail fast with a clear message.""" + try: + req = urllib.request.Request(f"{base}/info", method="GET") + urllib.request.urlopen(req, timeout=5) + except Exception: + print(json.dumps({ + "ok": False, + "error": f"Trellis is not available at {base}. The service may be switched off to save system resources. Start it before generating 3D models." + }, indent=2)) + sys.exit(1) + + +def generate(image_path, output=None, simplify=0.95, texture_size=1024, + seed=0, timeout=600): + """ + Generate a 3D model from an image. + + Pipeline: + 1. Start session + 2. Upload and preprocess image + 3. Generate 3D from image + 4. Extract GLB + 5. Download GLB file + + Args: + image_path: Path to the input image (PNG recommended) + output: Output .glb file path (default: auto-named) + simplify: Mesh simplification factor (0.9-0.98, default 0.95) + texture_size: Texture resolution (512-2048, default 1024) + seed: Random seed (default 0) + timeout: Max wait time per step in seconds + """ + base = get_base_url() + _check_available(base) + start_time = time.time() + + if not os.path.isfile(image_path): + print(json.dumps({"ok": False, "error": f"Image not found: {image_path}"}), indent=2) + sys.exit(1) + + if output is None: + name = os.path.splitext(os.path.basename(image_path))[0] + output = f"{name}.glb" + + # Step 1: Start session + print("Step 1/5: Starting session...", file=sys.stderr) + _call_api(base, "/start_session", [], timeout=30) + + # Step 2: Upload and preprocess image + print("Step 2/5: Uploading and preprocessing image...", file=sys.stderr) + uploaded_path = _upload_image(base, image_path) + file_ref = { + "path": uploaded_path, + "meta": {"_type": "gradio.FileData"} + } + preprocess_result = _call_api(base, "/preprocess_image_1", [file_ref], timeout=60) + + # _call_api returns the "data" array directly + if isinstance(preprocess_result, list) and len(preprocess_result) > 0: + preprocessed_ref = preprocess_result[0] + else: + preprocessed_ref = preprocess_result + + # Step 3: Get seed + print("Step 3/5: Generating 3D model...", file=sys.stderr) + seed_result = _call_api(base, "/get_seed", [True, seed], timeout=10) + if isinstance(seed_result, list) and seed_result: + actual_seed = seed_result[0] + else: + actual_seed = seed + + # Step 4: Image to 3D + # Parameters: image, multiimages, seed, ss_guidance, ss_steps, slat_guidance, slat_steps, algo + gen_result = _call_api(base, "/image_to_3d", [ + preprocessed_ref, # image + [], # multiimages (empty) + actual_seed, # seed + 7.5, # ss_guidance_strength + 12, # ss_sampling_steps + 3.0, # slat_guidance_strength + 12, # slat_sampling_steps + "stochastic", # multiimage_algo + ], timeout=timeout) + + # Step 5: Extract GLB + print("Step 4/5: Extracting GLB...", file=sys.stderr) + glb_result = _call_api(base, "/extract_glb", [simplify, texture_size], timeout=120) + + # _call_api returns the "data" array: [model_viewer_data, download_button_data] + glb_url = None + if isinstance(glb_result, list): + for item in glb_result: + if isinstance(item, dict): + url = item.get("url") or item.get("path") + if url: + glb_url = url + break + + if not glb_url: + print(json.dumps({ + "ok": False, + "error": "Could not extract GLB URL from response", + "response": glb_result + }, indent=2)) + sys.exit(1) + + # Step 6: Download + print("Step 5/5: Downloading GLB...", file=sys.stderr) + os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True) + file_size = _download_file(glb_url, output, base) + + elapsed = round(time.time() - start_time, 1) + print(json.dumps({ + "ok": True, + "file": output, + "size_bytes": file_size, + "simplify": simplify, + "texture_size": texture_size, + "seed": actual_seed, + "generation_time_s": elapsed, + "source_image": image_path + }, indent=2)) + + +def main(): + if len(sys.argv) < 2: + print("Usage:") + print(" trellis_connector.py health") + print(" trellis_connector.py generate image.png [--output model.glb] [--simplify 0.95] [--texture-size 1024] [--seed N] [--timeout N]") + sys.exit(1) + + cmd = sys.argv[1] + + if cmd == "health": + health() + elif cmd == "generate": + if len(sys.argv) < 3: + print("Error: image path required", file=sys.stderr) + sys.exit(1) + + image_path = sys.argv[2] + output = None + simplify = 0.95 + texture_size = 1024 + seed = 0 + timeout = 600 + + i = 3 + while i < len(sys.argv): + if sys.argv[i] == "--output" and i + 1 < len(sys.argv): + output = sys.argv[i + 1] + i += 2 + elif sys.argv[i] == "--simplify" and i + 1 < len(sys.argv): + simplify = float(sys.argv[i + 1]) + i += 2 + elif sys.argv[i] == "--texture-size" and i + 1 < len(sys.argv): + texture_size = int(sys.argv[i + 1]) + i += 2 + elif sys.argv[i] == "--seed" and i + 1 < len(sys.argv): + seed = int(sys.argv[i + 1]) + i += 2 + elif sys.argv[i] == "--timeout" and i + 1 < len(sys.argv): + timeout = int(sys.argv[i + 1]) + i += 2 + else: + print(f"Unknown argument: {sys.argv[i]}", file=sys.stderr) + sys.exit(1) + + generate(image_path, output=output, simplify=simplify, + texture_size=texture_size, seed=seed, timeout=timeout) + else: + print(f"Unknown command: {cmd}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main()