settled-reach/tests/run-visual

#!/usr/bin/env bash
# tests/run-visual: Visual regression test suite.
# Captures scenarios from tests/visual.json, compares against golden PNGs.
#
# Modes:
#   (no args)            Run all scenario golden comparisons (xvfb-wrapped)
#   --screenshot NAME    Ad-hoc single capture to .cache/screenshots/ (no xvfb)
#   --movie NAME         Flow capture to .cache/screenshots/ (no xvfb)
#   --update             Regenerate all goldens and stage for commit
#   --filter PATTERN     Accepted and ignored (compat with run-all)
#
# Exit: 0=pass (or skip), 1=fail, 2=error
# Stdout (golden mode): {"suite":"visual","total":N,"passed":N,"failed":N,"duration_ms":N}
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
CONFIG="$ROOT/tests/visual.json"

GODOT="$(command -v godot4 2>/dev/null || command -v godot 2>/dev/null || true)"
CACHE_DIR="$ROOT/.cache/screenshots"
DIFF_DIR="$ROOT/.cache/visual-diff"

MODE="golden"     # golden | screenshot | movie | update
TARGET=""
INTERVAL=""

# Server state for live scenarios
SERVER_BIN=""
SERVER_PID=""
SERVER_PORT=""

# Cleanup server on exit
trap '[[ -n "${SERVER_PID:-}" ]] && kill "$SERVER_PID" 2>/dev/null; wait "$SERVER_PID" 2>/dev/null || true' EXIT

# -- Parse args ----------------------------------------------------------------

while [[ $# -gt 0 ]]; do
    case "$1" in
        --screenshot) MODE="screenshot"; TARGET="${2:-atlas_GJ820Bc_Global}"; shift 2 ;;
        --movie)      MODE="movie";      TARGET="${2:-flow_dialogue}"; shift 2 ;;
        --update)     MODE="update"; shift ;;
        --filter)     shift 2 ;;  # Accept and ignore (run-all compat)
        --filter=*)   shift ;;
        --interval)   INTERVAL="${2:-3}"; shift 2 ;;
        *)            echo "Unknown argument: $1" >&2; exit 2 ;;
    esac
done

# -- Preflight -----------------------------------------------------------------

if [[ -z "$GODOT" ]]; then
    echo "Warning: Godot not found — skipping visual tests" >&2
    printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n'
    exit 0
fi

if ! python3 -c "pass" 2>/dev/null; then
    echo "Warning: Python 3 not found — skipping visual tests" >&2
    printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n'
    exit 0
fi

if [[ ! -f "$CONFIG" ]]; then
    echo "Error: $CONFIG not found" >&2
    exit 2
fi

# Read config values
GOLDEN_DIR="$ROOT/$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('golden_dir','client/tests/golden/visual'))")"
TOLERANCE="$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('tolerance', 5))")"
RESOLUTION="$(python3 -c "import json; c=json.load(open('$CONFIG')); r=c.get('resolution',[960,540]); print(f'{r[0]}x{r[1]}')")"

# Read scenario names from config
SCENARIOS=($(python3 -c "
import json
c = json.load(open('$CONFIG'))
for name in c.get('scenarios', {}):
    print(name)
"))

# -- Helpers -------------------------------------------------------------------

# Offscreen wrapper for EVERY capture path.
#
# Captures must never steal the desktop: this box is also the developer's own
# session. Until 2026-08-06 only the golden path even tried, via xvfb-run —
# which is not installed here, so it hit the "using visible window" fallback,
# and --screenshot/--movie never wrapped at all. Every capture grabbed focus.
#
# gamescope is preferred, for two independent reasons:
#   1. It renders on the REAL GPU. The goldens are pinned to this box's
#      Mesa/AMD output (T-1121 — they do not port across rendering stacks), and
#      xvfb-run would fall back to llvmpipe software rendering, shifting every
#      pixel. Offscreen must not mean a different renderer.
#   2. It is the only option that lets us SET the output size. cage is also
#      installed and also GPU-backed, but it is a kiosk compositor: it forces
#      its client to the headless output's default 1280x720 and silently
#      overrides --resolution (measured — a 960x540 request produced a 1280x720
#      PNG). A wrapper that quietly changes resolution is worse than none,
#      because RESOLUTION is load-bearing here: D-255's extent inversion makes
#      the shorter viewport axis span exactly one cell of the rung, so window
#      size decides how much world a rung shows. cage is deliberately not used.
CAPTURE_PREFIX=()
if command -v gamescope >/dev/null 2>&1; then
    CAPTURE_PREFIX=(gamescope --backend headless -W "${RESOLUTION%x*}" -H "${RESOLUTION#*x}" --)
elif command -v xvfb-run >/dev/null 2>&1; then
    echo "Note: gamescope not found — falling back to xvfb-run (software GL; goldens may drift)" >&2
    CAPTURE_PREFIX=(xvfb-run -a --server-args="-screen 0 ${RESOLUTION}x24")
else
    echo "Warning: no offscreen compositor (gamescope/xvfb-run) — capturing in a VISIBLE window" >&2
fi

godot_capture() {
    local mode_flag="$1"  # --scenario or --flow
    local name="$2"
    local output="$3"
    local extra_args=("${@:4}")

    "${CAPTURE_PREFIX[@]}" "$GODOT" --rendering-driver opengl3 --fixed-fps 60 \
        --resolution "$RESOLUTION" \
        --path "$ROOT/client" -s res://tests/visual_capture.gd -- \
        $mode_flag "$name" --output "$output" "${extra_args[@]}" 2>&1
}

# Retained as the golden-path name; offscreen handling now lives in
# godot_capture, so both paths get it.
xvfb_capture() {
    godot_capture "$@"
}

# -- Server lifecycle (live scenarios) -----------------------------------------

# Check if a scenario has "live": true in config
is_live_scenario() {
    python3 -c "
import json, sys
c = json.load(open('${CONFIG}'))
s = c.get('scenarios', {}).get('${1}', {})
sys.exit(0 if s.get('live') else 1)
"
}

# Build server binary (once, cached)
ensure_server_built() {
    if [[ -n "$SERVER_BIN" ]]; then return 0; fi
    echo "  Building server for live visual tests..."
    (cd "$ROOT/server" && cargo build --bin settled-reach-server 2>&1) || {
        echo "Error: server build failed" >&2
        return 1
    }
    SERVER_BIN="$ROOT/server/target/debug/settled-reach-server"
}

# Start server with --test-mode --port 0, parse LISTENING:{port}
start_server() {
    local stdout_log
    stdout_log=$(mktemp)
    "$SERVER_BIN" --test-mode --port 0 >"$stdout_log" 2>/dev/null &
    SERVER_PID=$!

    local attempts=0
    while [[ $attempts -lt 150 ]]; do
        if ! kill -0 "$SERVER_PID" 2>/dev/null; then
            echo "  Error: server exited unexpectedly" >&2
            rm -f "$stdout_log"
            SERVER_PID=""
            return 1
        fi
        if grep -q "^LISTENING:" "$stdout_log" 2>/dev/null; then
            SERVER_PORT=$(sed -n 's/^LISTENING://p' "$stdout_log")
            rm -f "$stdout_log"
            echo "  Server started: pid=$SERVER_PID port=$SERVER_PORT"
            return 0
        fi
        sleep 0.1
        attempts=$((attempts + 1))
    done

    echo "  Error: no LISTENING signal after 15s" >&2
    kill "$SERVER_PID" 2>/dev/null || true
    rm -f "$stdout_log"
    SERVER_PID=""
    return 1
}

# Stop server (called after each live capture; server may have exited on disconnect)
stop_server() {
    if [[ -n "$SERVER_PID" ]]; then
        kill "$SERVER_PID" 2>/dev/null || true
        wait "$SERVER_PID" 2>/dev/null || true
        SERVER_PID=""
        SERVER_PORT=""
    fi
}

# -- Screenshot mode -----------------------------------------------------------

if [[ "$MODE" == "screenshot" ]]; then
    mkdir -p "$CACHE_DIR"
    echo "Capturing scenario: $TARGET"
    if is_live_scenario "$TARGET"; then
        ensure_server_built || exit 2
        start_server || exit 2
        export SR_LIVE=1 SR_PORT="$SERVER_PORT"
    fi
    godot_capture --scenario "$TARGET" "$CACHE_DIR"
    stop_server
    unset SR_LIVE SR_PORT 2>/dev/null || true
    PNG="$CACHE_DIR/$TARGET.png"
    if [[ -f "$PNG" ]]; then
        echo "Screenshot: $PNG ($(stat -c%s "$PNG" 2>/dev/null || stat -f%z "$PNG") bytes)"
        # Advisory here rather than fatal — an ad-hoc capture of a rung that
        # genuinely renders nothing is a legitimate thing to want to look at
        # (that is how the empty deep rungs were found). But say so out loud,
        # because file size alone reads as success.
        "$ROOT/tooling/visual-blank-check" "$PNG" --quiet || true
    else
        echo "Error: capture failed — $PNG not found" >&2
        exit 1
    fi
    exit 0
fi

# -- Movie mode ----------------------------------------------------------------

if [[ "$MODE" == "movie" ]]; then
    mkdir -p "$CACHE_DIR"
    echo "Capturing flow: $TARGET"
    EXTRA=()
    [[ -n "$INTERVAL" ]] && EXTRA+=(--interval "$INTERVAL")
    godot_capture --flow "$TARGET" "$CACHE_DIR" "${EXTRA[@]}"

    FLOW_DIR="$CACHE_DIR/$TARGET"
    if [[ -d "$FLOW_DIR" ]]; then
        FRAME_COUNT=$(find "$FLOW_DIR" -name "*.png" | wc -l)
        echo "Flow: $FRAME_COUNT frames in $FLOW_DIR"

        # Generate contact sheet if visual-thumbnail is available
        if [[ -x "$ROOT/tooling/visual-thumbnail" ]]; then
            "$ROOT/tooling/visual-thumbnail" "$FLOW_DIR" --config "$CONFIG"
            SHEET="$FLOW_DIR/${TARGET}_sheet.png"
            [[ -f "$SHEET" ]] && echo "Contact sheet: $SHEET"
        fi
    else
        echo "Error: flow capture failed — $FLOW_DIR not found" >&2
        exit 1
    fi
    exit 0
fi

# -- Golden mode (default) / Update mode --------------------------------------

START_MS=$(date +%s%3N)
TOTAL=0
PASSED=0
FAILED=0

mkdir -p "$CACHE_DIR" "$DIFF_DIR"

if [[ "$MODE" == "update" ]]; then
    mkdir -p "$GOLDEN_DIR"
fi

for scenario in "${SCENARIOS[@]}"; do
    TOTAL=$((TOTAL + 1))
    echo "--- $scenario ---"

    # Start server for live scenarios
    IS_LIVE=false
    if is_live_scenario "$scenario"; then
        IS_LIVE=true
        ensure_server_built || { FAILED=$((FAILED + 1)); continue; }
        start_server || { FAILED=$((FAILED + 1)); continue; }
        export SR_LIVE=1 SR_PORT="$SERVER_PORT"
    fi

    # Capture
    set +e
    CAPTURE_OUT=$(xvfb_capture --scenario "$scenario" "$CACHE_DIR" 2>&1)
    CAPTURE_RC=$?
    set -e

    # Stop server after capture (server exits on client disconnect anyway)
    if [[ "$IS_LIVE" == "true" ]]; then
        unset SR_LIVE SR_PORT 2>/dev/null || true
        stop_server
    fi

    CAPTURED="$CACHE_DIR/$scenario.png"

    if [[ $CAPTURE_RC -ne 0 ]] || [[ ! -f "$CAPTURED" ]]; then
        echo "  FAIL: capture failed (exit $CAPTURE_RC)" >&2
        echo "$CAPTURE_OUT" >&2
        FAILED=$((FAILED + 1))
        continue
    fi

    # Verify non-empty
    if [[ ! -s "$CAPTURED" ]]; then
        echo "  FAIL: captured PNG is empty" >&2
        FAILED=$((FAILED + 1))
        continue
    fi

    # Verify the renderer actually drew something.
    #
    # "Non-empty file" was the only content check until 2026-08-06, and a blank
    # screen is a perfectly valid ~19 KB PNG. Worse, once a blank capture was
    # recorded as a golden, every later blank capture matched it at 0.0% and
    # the scenario PASSED — atlas_GJ338Bd_Block and atlas_GJ445c-m1_Chunk were
    # green against blank goldens. A test that cannot fail is worse than no
    # test, because it is counted as coverage.
    #
    # Checked in BOTH modes, and the update mode matters most: refusing to
    # RECORD a blank golden is what stops the trap being re-armed.
    set +e
    BLANK_OUT=$("$ROOT/tooling/visual-blank-check" "$CAPTURED" 2>&1)
    BLANK_RC=$?
    set -e
    if [[ $BLANK_RC -ne 0 ]]; then
        echo "  FAIL: $BLANK_OUT" >&2
        FAILED=$((FAILED + 1))
        continue
    fi

    if [[ "$MODE" == "update" ]]; then
        cp "$CAPTURED" "$GOLDEN_DIR/$scenario.png"
        echo "  Updated golden: $GOLDEN_DIR/$scenario.png"
        PASSED=$((PASSED + 1))
    else
        GOLDEN="$GOLDEN_DIR/$scenario.png"
        if [[ ! -f "$GOLDEN" ]]; then
            echo "  FAIL: golden not found — run 'make visual-update' first" >&2
            FAILED=$((FAILED + 1))
            continue
        fi

        # Compare
        set +e
        DIFF_OUT=$("$ROOT/tooling/visual-diff" "$GOLDEN" "$CAPTURED" \
            --tolerance "$TOLERANCE" \
            --diff-output "$DIFF_DIR/$scenario-diff.png" \
            --config "$CONFIG" 2>&1)
        DIFF_RC=$?
        set -e

        if [[ $DIFF_RC -eq 0 ]]; then
            echo "  $DIFF_OUT"
            PASSED=$((PASSED + 1))
        elif [[ $DIFF_RC -eq 1 ]]; then
            echo "  $DIFF_OUT"
            echo "  Diff image: $DIFF_DIR/$scenario-diff.png"
            FAILED=$((FAILED + 1))
        else
            echo "  ERROR: visual-diff failed (exit $DIFF_RC)" >&2
            echo "  $DIFF_OUT" >&2
            FAILED=$((FAILED + 1))
        fi
    fi
done

END_MS=$(date +%s%3N)
DURATION_MS=$((END_MS - START_MS))

if [[ "$MODE" == "update" ]]; then
    # Stage golden files
    cd "$ROOT"
    git add "$GOLDEN_DIR/" 2>/dev/null || true
    echo ""
    echo "=== Visual goldens updated ($PASSED of $TOTAL) ==="
    echo "Review with: git diff --cached -- $GOLDEN_DIR/"
fi

printf '{"suite":"visual","total":%d,"passed":%d,"failed":%d,"duration_ms":%d}\n' \
    "$TOTAL" "$PASSED" "$FAILED" "$DURATION_MS"

[[ $FAILED -gt 0 ]] && exit 1
exit 0