add tree-sitter syntax highlighting via dart:ffi
Vendor libtree-sitter.so (v0.26.8, wasmtime 44.0 statically linked) and call its C API through dart:ffi. Grammar WASM files are loaded by tree-sitter's embedded wasmtime engine via ts_wasm_store_load_language. 48 grammars compiled with tree-sitter build --wasm, 48 highlight queries (42 from upstream repos, 6 written in-house for comment, erlang, gdscript, gitignore, hcl, hlsl, swift). SQLite grammar replaces full SQL (974K-line parser exhausted RAM at compile time). Removes wasm_run and wasm_run_flutter — they downloaded a 22 MB binary from GitHub at first launch, violating the no-network-on- default-launch-path policy. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1,87 @@
|
||||
// Tree-sitter WASM bridge — compiled per grammar into a self-contained
|
||||
// WASM module. Exports: init, set_query, parse_and_highlight,
|
||||
// capture_count, capture_name, ts_alloc, ts_dealloc.
|
||||
|
||||
#include "tree_sitter/api.h"
|
||||
#include <stdlib.h>
|
||||
#include <stdint.h>
|
||||
|
||||
// Linked at compile time via -DGRAMMAR_FN=tree_sitter_<lang>.
|
||||
extern const TSLanguage *GRAMMAR_FN(void);
|
||||
|
||||
static TSParser *g_parser = NULL;
|
||||
static TSQuery *g_query = NULL;
|
||||
static TSQueryCursor *g_cursor = NULL;
|
||||
|
||||
// 12-byte packed span: start_byte, end_byte, capture_index.
|
||||
typedef struct { uint32_t start; uint32_t end; uint32_t capture; } Span;
|
||||
|
||||
__attribute__((export_name("init")))
|
||||
int32_t init(void) {
|
||||
g_parser = ts_parser_new();
|
||||
if (!g_parser) return -1;
|
||||
if (!ts_parser_set_language(g_parser, GRAMMAR_FN())) return -2;
|
||||
g_cursor = ts_query_cursor_new();
|
||||
if (!g_cursor) return -3;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Returns 0 on success, or (error_offset + 1) on failure.
|
||||
__attribute__((export_name("set_query")))
|
||||
int32_t set_query(const char *src, uint32_t len) {
|
||||
if (g_query) { ts_query_delete(g_query); g_query = NULL; }
|
||||
uint32_t error_offset;
|
||||
TSQueryError error_type;
|
||||
g_query = ts_query_new(GRAMMAR_FN(), src, len, &error_offset, &error_type);
|
||||
return g_query ? 0 : (int32_t)(error_offset + 1);
|
||||
}
|
||||
|
||||
// Parse source and run the active query. Writes Span structs into |out|.
|
||||
// Returns the number of spans written.
|
||||
__attribute__((export_name("parse_and_highlight")))
|
||||
uint32_t parse_and_highlight(
|
||||
const char *src, uint32_t src_len,
|
||||
Span *out, uint32_t max_spans
|
||||
) {
|
||||
if (!g_parser || !g_query || !g_cursor) return 0;
|
||||
|
||||
TSTree *tree = ts_parser_parse_string(g_parser, NULL, src, src_len);
|
||||
if (!tree) return 0;
|
||||
|
||||
TSNode root = ts_tree_root_node(tree);
|
||||
ts_query_cursor_exec(g_cursor, g_query, root);
|
||||
|
||||
TSQueryMatch match;
|
||||
uint32_t count = 0;
|
||||
while (ts_query_cursor_next_match(g_cursor, &match) && count < max_spans) {
|
||||
for (uint16_t i = 0; i < match.capture_count && count < max_spans; i++) {
|
||||
TSQueryCapture cap = match.captures[i];
|
||||
out[count].start = ts_node_start_byte(cap.node);
|
||||
out[count].end = ts_node_end_byte(cap.node);
|
||||
out[count].capture = cap.index;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
ts_tree_delete(tree);
|
||||
return count;
|
||||
}
|
||||
|
||||
__attribute__((export_name("capture_count")))
|
||||
uint32_t capture_count(void) {
|
||||
return g_query ? ts_query_capture_count(g_query) : 0;
|
||||
}
|
||||
|
||||
// Returns a pointer into WASM linear memory. Caller reads |*out_len|
|
||||
// bytes starting at the returned address.
|
||||
__attribute__((export_name("capture_name")))
|
||||
const char *capture_name(uint32_t index, uint32_t *out_len) {
|
||||
if (!g_query) { *out_len = 0; return NULL; }
|
||||
return ts_query_capture_name_for_id(g_query, index, out_len);
|
||||
}
|
||||
|
||||
__attribute__((export_name("ts_alloc")))
|
||||
void *ts_alloc(uint32_t size) { return malloc(size); }
|
||||
|
||||
__attribute__((export_name("ts_dealloc")))
|
||||
void ts_dealloc(void *ptr) { free(ptr); }
|
||||
Executable
+162
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env bash
|
||||
# Compile tree-sitter core + each grammar + bridge.c into self-contained
|
||||
# WASM modules. Requires wasi-sdk (cached by `tree-sitter build --wasm`).
|
||||
#
|
||||
# Usage:
|
||||
# ./tools/ts-wasm/build.sh # build all grammars
|
||||
# ./tools/ts-wasm/build.sh dart rust # build specific grammars
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
WASI_SDK="${WASI_SDK:-$HOME/.cache/tree-sitter/wasi-sdk}"
|
||||
TS_CORE="${TS_CORE:-/var/mnt/data/projects/treesitter/tree-sitter}"
|
||||
GRAMMARS_ROOT="${GRAMMARS_ROOT:-/var/mnt/data/projects/treesitter}"
|
||||
OUT_DIR="$REPO_ROOT/app/assets/grammars"
|
||||
QUERIES_DIR="$REPO_ROOT/app/assets/queries"
|
||||
|
||||
CC="$WASI_SDK/bin/clang"
|
||||
CXX="$WASI_SDK/bin/clang++"
|
||||
SYSROOT="$WASI_SDK/share/wasi-sysroot"
|
||||
|
||||
CFLAGS=(
|
||||
--sysroot="$SYSROOT"
|
||||
--target=wasm32-wasip1
|
||||
-O2 -flto
|
||||
-fno-exceptions
|
||||
-I"$TS_CORE/lib/include"
|
||||
-I"$TS_CORE/lib/src"
|
||||
-DTREE_SITTER_HIDE_SYMBOLS
|
||||
)
|
||||
|
||||
LDFLAGS=(
|
||||
-Wl,--no-entry
|
||||
-Wl,--export=init
|
||||
-Wl,--export=set_query
|
||||
-Wl,--export=parse_and_highlight
|
||||
-Wl,--export=capture_count
|
||||
-Wl,--export=capture_name
|
||||
-Wl,--export=ts_alloc
|
||||
-Wl,--export=ts_dealloc
|
||||
-Wl,--export=__heap_base
|
||||
-Wl,--strip-all
|
||||
-Wl,--gc-sections
|
||||
)
|
||||
|
||||
mkdir -p "$OUT_DIR" "$QUERIES_DIR"
|
||||
|
||||
# Grammar source-dir resolution. Multi-grammar repos (markdown, php,
|
||||
# typescript, xml) nest the actual grammar under a subdirectory.
|
||||
grammar_src_dir() {
|
||||
local name="$1" base="$GRAMMARS_ROOT/tree-sitter-$name"
|
||||
# Multi-grammar repos: check for a nested dir matching the grammar name.
|
||||
for sub in "$base/$name" "$base/tree-sitter-$name"; do
|
||||
[ -f "$sub/src/parser.c" ] && echo "$sub" && return
|
||||
done
|
||||
# Fallback: repo root.
|
||||
[ -f "$base/src/parser.c" ] && echo "$base" && return
|
||||
echo >&2 "error: no parser.c for $name"; return 1
|
||||
}
|
||||
|
||||
# Derive the C function name (tree_sitter_<id>) from parser.c.
|
||||
grammar_fn_name() {
|
||||
grep -oP 'TSLanguage \*\K(tree_sitter_\w+)(?=\(void\))' "$1/src/parser.c" | head -1
|
||||
}
|
||||
|
||||
# Find highlight queries, preferring the grammar's own queries/ dir.
|
||||
copy_queries() {
|
||||
local name="$1" src_dir="$2"
|
||||
# Try grammar-specific queries first, then repo-level.
|
||||
local q=""
|
||||
for candidate in \
|
||||
"$src_dir/queries/highlights.scm" \
|
||||
"$GRAMMARS_ROOT/tree-sitter-$name/queries/highlights.scm" \
|
||||
"$GRAMMARS_ROOT/tree-sitter-$name/queries/$name/highlights.scm" \
|
||||
"$GRAMMARS_ROOT/tree-sitter-$name/queries-src/highlights.scm"; do
|
||||
[ -f "$candidate" ] && q="$candidate" && break
|
||||
done
|
||||
[ -n "$q" ] && cp "$q" "$QUERIES_DIR/$name.scm" || true
|
||||
}
|
||||
|
||||
build_grammar() {
|
||||
local name="$1"
|
||||
local src_dir; src_dir="$(grammar_src_dir "$name")" || return 1
|
||||
local fn; fn="$(grammar_fn_name "$src_dir")"
|
||||
[ -z "$fn" ] && echo >&2 "error: can't find function name for $name" && return 1
|
||||
|
||||
local c_sources=( "$SCRIPT_DIR/bridge.c" "$TS_CORE/lib/src/lib.c" "$src_dir/src/parser.c" )
|
||||
local cxx_sources=()
|
||||
|
||||
# Add external scanner.
|
||||
if [ -f "$src_dir/src/scanner.c" ]; then
|
||||
c_sources+=( "$src_dir/src/scanner.c" )
|
||||
fi
|
||||
if [ -f "$src_dir/src/scanner.cc" ]; then
|
||||
cxx_sources+=( "$src_dir/src/scanner.cc" )
|
||||
fi
|
||||
|
||||
local mem; mem="$(free -h | awk '/Mem:/{print $3" used / "$7" avail"}')"
|
||||
echo " $name ($fn) [$mem]"
|
||||
|
||||
local tmpdir; tmpdir="$(mktemp -d)"
|
||||
trap "rm -rf '$tmpdir'" RETURN
|
||||
|
||||
# Drop -flto for C++ grammars — wasm-ld LTO on C++ eats 20 GB+.
|
||||
local compile_flags=("${CFLAGS[@]}")
|
||||
if [ ${#cxx_sources[@]} -gt 0 ]; then
|
||||
compile_flags=("${compile_flags[@]/-flto/}")
|
||||
fi
|
||||
|
||||
# Compile C sources.
|
||||
local objs=()
|
||||
local i=0
|
||||
for src in "${c_sources[@]}"; do
|
||||
$CC "${compile_flags[@]}" -DGRAMMAR_FN="$fn" -I"$src_dir/src" \
|
||||
-c "$src" -o "$tmpdir/$i.o" 2>&1
|
||||
objs+=( "$tmpdir/$i.o" )
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
# Compile C++ sources (if any).
|
||||
for src in "${cxx_sources[@]}"; do
|
||||
$CXX "${compile_flags[@]}" -DGRAMMAR_FN="$fn" -I"$src_dir/src" \
|
||||
-fno-rtti -c "$src" -o "$tmpdir/$i.o" 2>&1
|
||||
objs+=( "$tmpdir/$i.o" )
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
# Link.
|
||||
local linker="$CC"
|
||||
[ ${#cxx_sources[@]} -gt 0 ] && linker="$CXX"
|
||||
$linker "${compile_flags[@]}" "${objs[@]}" "${LDFLAGS[@]}" \
|
||||
-o "$OUT_DIR/$name.wasm" 2>&1
|
||||
|
||||
copy_queries "$name" "$src_dir"
|
||||
}
|
||||
|
||||
# Determine which grammars to build.
|
||||
if [ $# -gt 0 ]; then
|
||||
targets=("$@")
|
||||
else
|
||||
targets=()
|
||||
for d in "$GRAMMARS_ROOT"/tree-sitter-*/; do
|
||||
g="$(basename "$d" | sed 's/^tree-sitter-//')"
|
||||
targets+=("$g")
|
||||
done
|
||||
fi
|
||||
|
||||
echo "Building ${#targets[@]} grammars..."
|
||||
failed=()
|
||||
for g in "${targets[@]}"; do
|
||||
if ! build_grammar "$g"; then
|
||||
failed+=("$g")
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Built $((${#targets[@]} - ${#failed[@]})) / ${#targets[@]} grammars."
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
echo "Failed: ${failed[*]}"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user