settled-reach/tests/run-godot

#!/usr/bin/env bash
# tests/run-godot: Run Godot client test suite via gdUnit4 (D-030)
#
# Exit: 0 = all pass, non-zero = failure.
# Stdout: JSON summary on ONE line, plus a pointer to the full log.
#   {"suite":"godot","total":N,"passed":N,"failed":N,"skipped":N,"duration_ms":N,"log":"..."}
#   (on a harness failure: same JSON + "harness_error":"load_error"|"no_tests"; exit 2)
#   (on timeout: same JSON + "timeout":true, "timeout_sec":N; exit code 124)
# Stderr: a short hint line pointing at the log. The Godot/gdUnit4 output
#   does NOT stream to stdout or stderr — it is captured to the log file.
#   This is intentional: streaming 20k+ lines of test log into an LLM
#   caller's context is unworkable. Inspect the log with the commands the
#   hint line suggests.
#
# --filter <stem|filename|res-path>: narrow the test run to a single file.
set -euo pipefail

# Hard wall-clock cap. Sprint 36 lost an hour to a hung test suite that
# silently consumed CPU forever. 300s is generous for the full suite
# (which currently runs in ~60s) and well above the slowest single suite
# (~40s for the compositor build). If you need longer for an unusual
# workload (fixture regen, etc.), prefer adding a dedicated script over
# extending this cap — the cap is the point.
TIMEOUT_SEC=300

# Per-process log path. PID suffix prevents concurrent runs across worktrees
# from clobbering each other's logs and producing summary JSON that mixes
# counts from different suites (R2-Hoshe-2). The actual path is echoed back
# via the JSON "log" field and the stderr hint line, so callers don't need
# to predict it.
LOG_FILE="/tmp/sr-run-godot.$$.log"

FILTER=""
while [[ $# -gt 0 ]]; do
    case "$1" in
        --filter) FILTER="${2:-}"; shift 2 ;;
        --filter=*) FILTER="${1#--filter=}"; shift ;;
        *) echo "Unknown argument: $1" >&2; exit 2 ;;
    esac
done

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

GODOT=$(command -v godot4 2>/dev/null || command -v godot 2>/dev/null || echo "")
if [[ -z "$GODOT" ]]; then
    printf '{"suite":"godot","total":0,"passed":0,"failed":0,"duration_ms":0,"error":"godot not found in PATH"}\n'
    exit 1
fi

# Resolve the test target: directory or specific file
if [[ -n "$FILTER" ]]; then
    # Support bare name (test_protocol) or full path (test_protocol.gd)
    if [[ "$FILTER" == res://* ]]; then
        TEST_TARGET="$FILTER"
    elif [[ "$FILTER" == *.gd ]]; then
        TEST_TARGET="res://tests/$FILTER"
    else
        TEST_TARGET="res://tests/${FILTER}.gd"
    fi
else
    TEST_TARGET="res://tests/"
fi

START_MS=$(date +%s%3N)

# Redirect Godot+gdUnit4 output to the log file. Nothing streams to the
# caller — the summary JSON (stdout) and the hint line (stderr) are the
# only things the caller ever sees. See header comment for rationale.
set +e
timeout --foreground --kill-after=10 "$TIMEOUT_SEC" \
    "$GODOT" --headless --path "$REPO_ROOT/client" \
        -s res://addons/gdUnit4/bin/GdUnitCmdTool.gd \
        --ignoreHeadlessMode \
        -c \
        -a "$TEST_TARGET" \
    > "$LOG_FILE" 2>&1
EXIT_CODE=$?
set -e

END_MS=$(date +%s%3N)
DURATION_MS=$((END_MS - START_MS))

# timeout(1) exit code 124 = wall clock exceeded; 137 = needed SIGKILL.
TIMED_OUT=false
if [[ "$EXIT_CODE" -eq 124 || "$EXIT_CODE" -eq 137 ]]; then
    TIMED_OUT=true
fi

# gdUnit4 prints one "Statistics:" line PER SUITE and then a single
# "Overall Summary:" line whose numbers are the sum of all of them.
#
# BUG FIXED 2026-07-27: the old pattern matched BOTH shapes and summed all of
# them, so every reported total was exactly DOUBLE — the real number plus the
# summary that already contained it. A full run reported 3,660 tests against an
# actual 1,830, and a 26-test suite reported 52. It had been wrong for as long
# as the summary line has existed, and it was invisible precisely because it
# doubled uniformly: nothing ever looked inconsistent, only large.
#
# Prefer the Overall Summary — it is gdUnit4's own arithmetic over the whole
# run, so it cannot disagree with itself. Fall back to summing per-suite lines
# only if it is absent (older gdUnit4, or a run that died mid-way).
# gdUnit4 colourises its output, and the escape sequences sit BETWEEN the
# fields of the summary line — so every pattern here must run against a
# de-ANSI'd copy. Matching the raw log silently falls through to the weaker
# "Executed test cases" fallback, which cannot see skips and reported a fully
# skipped suite as 26 FAILED (caught 2026-07-27, immediately after the
# double-count fix — same log, second lie).
CLEAN_LOG="${LOG_FILE%.log}.clean.log"
sed 's/\x1b\[[0-9;]*m//g' "$LOG_FILE" > "$CLEAN_LOG" 2>/dev/null || cp "$LOG_FILE" "$CLEAN_LOG"

TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
SUMMARY_LINE=$(grep -oE "Overall Summary: [0-9]+ test cases \| [0-9]+ errors \| [0-9]+ failures \| [0-9]+ flaky \| [0-9]+ skipped" "$CLEAN_LOG" | tail -1 || true)
if [[ -z "$SUMMARY_LINE" ]]; then
    SUMMARY_LINE=$(grep -oE "Statistics: [0-9]+ test cases \| [0-9]+ errors \| [0-9]+ failures \| [0-9]+ flaky \| [0-9]+ skipped" "$CLEAN_LOG" || true)
fi
if [[ -n "$SUMMARY_LINE" ]]; then
    _sum() { echo "$SUMMARY_LINE" | grep -oE "[0-9]+ $1" | grep -oE '^[0-9]+' | awk '{s+=$1} END {print s+0}'; }
    TOTAL=$(echo "$SUMMARY_LINE" | grep -oE '[0-9]+ test cases' | grep -oE '^[0-9]+' | awk '{s+=$1} END {print s+0}')
    FAILED=$(( $(_sum errors) + $(_sum failures) ))
    SKIPPED=$(_sum skipped)
    # A skipped test is NOT a passing test. Counting it as one is the same
    # false-green shape the guards below exist to prevent — and it matters now
    # that whole suites are deliberately skipped (test_character_visual_sprint28).
    PASSED=$(( TOTAL - FAILED - SKIPPED ))
fi

# Fallback: parse "Executed test cases : (X/N)" for total if stats parse failed
if [[ "$TOTAL" -eq 0 ]]; then
    EXEC_LINE=$(grep -oE "Executed test cases : \([0-9]+/[0-9]+\)" "$CLEAN_LOG" | tail -1 || true)
    if [[ -n "$EXEC_LINE" ]]; then
        TOTAL=$(echo "$EXEC_LINE" | grep -oE '/[0-9]+\)' | grep -oE '[0-9]+')
        PASSED=$(echo "$EXEC_LINE" | grep -oE '\([0-9]+/' | grep -oE '[0-9]+')
        FAILED=$(( TOTAL - PASSED ))
    fi
fi

# --- Harness-integrity guards (2026-07-26) -------------------------------
# A test file that fails to PARSE never runs, and gdUnit4 reports whatever
# DID run as a clean pass — so a broken suite reads as success. Both halves
# of that bit on the same day:
#   * a filtered run printed {"total":0,...} followed by "Tests passed";
#   * a full run printed 3610 passed / 0 failed while silently dropping an
#     entire suite whose parse error had gone unnoticed for hours.
# Neither is a test RESULT, so neither may be reported as one. A run that
# executed zero tests is never a pass, and a run that could not load a suite
# is a harness failure regardless of how many other suites went green.
LOAD_ERROR_COUNT=$(grep -c 'Failed to load script' "$CLEAN_LOG" 2>/dev/null || true)
LOAD_ERROR_LIST=$(grep -oE 'Failed to load script "[^"]+"' "$CLEAN_LOG" 2>/dev/null | sort -u || true)
HARNESS_ERROR=""
if [[ "${LOAD_ERROR_COUNT:-0}" -gt 0 ]]; then
    HARNESS_ERROR="load_error"
elif [[ "$TOTAL" -eq 0 && "$TIMED_OUT" != "true" ]]; then
    HARNESS_ERROR="no_tests"
fi

LOG_LINES=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0)

# Single-line JSON summary on stdout — machine-parseable, small.
if [[ "$TIMED_OUT" == "true" ]]; then
    printf '{"suite":"godot","total":%d,"passed":%d,"failed":%d,"duration_ms":%d,"timeout":true,"timeout_sec":%d,"log":"%s"}\n' \
        "${TOTAL:-0}" "${PASSED:-0}" "${FAILED:-0}" "$DURATION_MS" "$TIMEOUT_SEC" "$LOG_FILE"
elif [[ -n "$HARNESS_ERROR" ]]; then
    printf '{"suite":"godot","total":%d,"passed":%d,"failed":%d,"duration_ms":%d,"harness_error":"%s","log":"%s"}\n' \
        "${TOTAL:-0}" "${PASSED:-0}" "${FAILED:-0}" "$DURATION_MS" "$HARNESS_ERROR" "$LOG_FILE"
else
    printf '{"suite":"godot","total":%d,"passed":%d,"failed":%d,"skipped":%d,"duration_ms":%d,"log":"%s"}\n' \
        "${TOTAL:-0}" "${PASSED:-0}" "${FAILED:-0}" "${SKIPPED:-0}" "$DURATION_MS" "$LOG_FILE"
fi

# Hint line on stderr. Stays short. LLM callers: read this literally.
if [[ "$TIMED_OUT" == "true" ]]; then
    cat >&2 <<EOF
TEST_TIMEOUT: tests/run-godot exceeded ${TIMEOUT_SEC}s wall-clock cap (killed=${EXIT_CODE}).
  filter=${FILTER:-<none>}  target=${TEST_TARGET}  log=${LOG_FILE} (${LOG_LINES} lines)
  This is NOT a regular test failure — the process was force-terminated.
  A hung test almost always means a cleanup hook froze (e.g. after_test
  freeing GdUnit4 infrastructure) or an assertion waits on a signal that
  never fires. Bisect:
    grep -E 'STARTED|PASSED|FAILED' ${LOG_FILE} | tail -20
  The last STARTED without a matching PASSED/FAILED is the hang site.
EOF
elif [[ "$HARNESS_ERROR" == "load_error" ]]; then
    cat >&2 <<EOF
HARNESS_ERROR: a test suite failed to LOAD — this is not a test result.
  ${LOAD_ERROR_LIST}
  Those suites did not run at all. Any pass count above EXCLUDES them, so a
  green number here would be a lie. Usually a parse error (arity change, a
  renamed symbol, a stale call signature) in the listed file. Find it with:
    grep -n 'Parse Error' ${LOG_FILE} | head
  log=${LOG_FILE} (${LOG_LINES} lines)
EOF
elif [[ "$HARNESS_ERROR" == "no_tests" ]]; then
    cat >&2 <<EOF
HARNESS_ERROR: zero tests executed — this is not a pass.
  filter=${FILTER:-<none>}  target=${TEST_TARGET}  log=${LOG_FILE} (${LOG_LINES} lines)
  Either the filter matched no suite (check the name), or the target failed to
  load. A run that executes nothing can never be reported as success.
EOF
elif [[ "${FAILED:-0}" -gt 0 ]]; then
    cat >&2 <<EOF
Tests finished with failures. log=${LOG_FILE} (${LOG_LINES} lines)
  Summarize failures per suite:
    sed 's/\x1b\[[0-9;]*m//g' ${LOG_FILE} | grep -E 'FAILED [0-9]+ms\$' | awk -F'>' '{print \$1}' | sort | uniq -c | sort -rn
  First failure block with context:
    sed 's/\x1b\[[0-9;]*m//g' ${LOG_FILE} | grep -n 'FAILED\|Expecting\|Godot Runtime Error' | head -40
EOF
else
    echo "Tests passed. log=${LOG_FILE} (${LOG_LINES} lines)" >&2
fi

# A harness failure must never inherit gdUnit4's exit code — it exits 0 both
# when a suite fails to load and when the filter matches nothing, which is the
# whole reason these two states were invisible.
if [[ -n "$HARNESS_ERROR" && "$EXIT_CODE" -eq 0 ]]; then
    exit 2
fi

exit $EXIT_CODE