#!/usr/bin/env bash # Compile tree-sitter core + each grammar + bridge.c into self-contained # WASM modules. Requires wasi-sdk (cached by `tree-sitter build --wasm`). # # Usage: # ./tools/ts-wasm/build.sh # build all grammars # ./tools/ts-wasm/build.sh dart rust # build specific grammars set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" WASI_SDK="${WASI_SDK:-$HOME/.cache/tree-sitter/wasi-sdk}" TS_CORE="${TS_CORE:-/var/mnt/data/projects/treesitter/tree-sitter}" GRAMMARS_ROOT="${GRAMMARS_ROOT:-/var/mnt/data/projects/treesitter}" OUT_DIR="$REPO_ROOT/app/assets/grammars" QUERIES_DIR="$REPO_ROOT/app/assets/queries" CC="$WASI_SDK/bin/clang" CXX="$WASI_SDK/bin/clang++" SYSROOT="$WASI_SDK/share/wasi-sysroot" CFLAGS=( --sysroot="$SYSROOT" --target=wasm32-wasip1 -O2 -flto -fno-exceptions -I"$TS_CORE/lib/include" -I"$TS_CORE/lib/src" -DTREE_SITTER_HIDE_SYMBOLS ) LDFLAGS=( -Wl,--no-entry -Wl,--export=init -Wl,--export=set_query -Wl,--export=parse_and_highlight -Wl,--export=capture_count -Wl,--export=capture_name -Wl,--export=ts_alloc -Wl,--export=ts_dealloc -Wl,--export=__heap_base -Wl,--strip-all -Wl,--gc-sections ) mkdir -p "$OUT_DIR" "$QUERIES_DIR" # Grammar source-dir resolution. Multi-grammar repos (markdown, php, # typescript, xml) nest the actual grammar under a subdirectory. grammar_src_dir() { local name="$1" base="$GRAMMARS_ROOT/tree-sitter-$name" # Multi-grammar repos: check for a nested dir matching the grammar name. for sub in "$base/$name" "$base/tree-sitter-$name"; do [ -f "$sub/src/parser.c" ] && echo "$sub" && return done # Fallback: repo root. [ -f "$base/src/parser.c" ] && echo "$base" && return echo >&2 "error: no parser.c for $name"; return 1 } # Derive the C function name (tree_sitter_) from parser.c. grammar_fn_name() { grep -oP 'TSLanguage \*\K(tree_sitter_\w+)(?=\(void\))' "$1/src/parser.c" | head -1 } # Find highlight queries, preferring the grammar's own queries/ dir. copy_queries() { local name="$1" src_dir="$2" # Try grammar-specific queries first, then repo-level. local q="" for candidate in \ "$src_dir/queries/highlights.scm" \ "$GRAMMARS_ROOT/tree-sitter-$name/queries/highlights.scm" \ "$GRAMMARS_ROOT/tree-sitter-$name/queries/$name/highlights.scm" \ "$GRAMMARS_ROOT/tree-sitter-$name/queries-src/highlights.scm"; do [ -f "$candidate" ] && q="$candidate" && break done [ -n "$q" ] && cp "$q" "$QUERIES_DIR/$name.scm" || true } build_grammar() { local name="$1" local src_dir; src_dir="$(grammar_src_dir "$name")" || return 1 local fn; fn="$(grammar_fn_name "$src_dir")" [ -z "$fn" ] && echo >&2 "error: can't find function name for $name" && return 1 local c_sources=( "$SCRIPT_DIR/bridge.c" "$TS_CORE/lib/src/lib.c" "$src_dir/src/parser.c" ) local cxx_sources=() # Add external scanner. if [ -f "$src_dir/src/scanner.c" ]; then c_sources+=( "$src_dir/src/scanner.c" ) fi if [ -f "$src_dir/src/scanner.cc" ]; then cxx_sources+=( "$src_dir/src/scanner.cc" ) fi local mem; mem="$(free -h | awk '/Mem:/{print $3" used / "$7" avail"}')" echo " $name ($fn) [$mem]" local tmpdir; tmpdir="$(mktemp -d)" trap "rm -rf '$tmpdir'" RETURN # Drop -flto for C++ grammars — wasm-ld LTO on C++ eats 20 GB+. local compile_flags=("${CFLAGS[@]}") if [ ${#cxx_sources[@]} -gt 0 ]; then compile_flags=("${compile_flags[@]/-flto/}") fi # Compile C sources. local objs=() local i=0 for src in "${c_sources[@]}"; do $CC "${compile_flags[@]}" -DGRAMMAR_FN="$fn" -I"$src_dir/src" \ -c "$src" -o "$tmpdir/$i.o" 2>&1 objs+=( "$tmpdir/$i.o" ) i=$((i + 1)) done # Compile C++ sources (if any). for src in "${cxx_sources[@]}"; do $CXX "${compile_flags[@]}" -DGRAMMAR_FN="$fn" -I"$src_dir/src" \ -fno-rtti -c "$src" -o "$tmpdir/$i.o" 2>&1 objs+=( "$tmpdir/$i.o" ) i=$((i + 1)) done # Link. local linker="$CC" [ ${#cxx_sources[@]} -gt 0 ] && linker="$CXX" $linker "${compile_flags[@]}" "${objs[@]}" "${LDFLAGS[@]}" \ -o "$OUT_DIR/$name.wasm" 2>&1 copy_queries "$name" "$src_dir" } # Determine which grammars to build. if [ $# -gt 0 ]; then targets=("$@") else targets=() for d in "$GRAMMARS_ROOT"/tree-sitter-*/; do g="$(basename "$d" | sed 's/^tree-sitter-//')" targets+=("$g") done fi echo "Building ${#targets[@]} grammars..." failed=() for g in "${targets[@]}"; do if ! build_grammar "$g"; then failed+=("$g") fi done echo "" echo "Built $((${#targets[@]} - ${#failed[@]})) / ${#targets[@]} grammars." if [ ${#failed[@]} -gt 0 ]; then echo "Failed: ${failed[*]}" exit 1 fi