add tree-sitter syntax highlighting via dart:ffi

Vendor libtree-sitter.so (v0.26.8, wasmtime 44.0 statically linked)
and call its C API through dart:ffi. Grammar WASM files are loaded by
tree-sitter's embedded wasmtime engine via ts_wasm_store_load_language.

48 grammars compiled with tree-sitter build --wasm, 48 highlight
queries (42 from upstream repos, 6 written in-house for comment,
erlang, gdscript, gitignore, hcl, hlsl, swift). SQLite grammar
replaces full SQL (974K-line parser exhausted RAM at compile time).

Removes wasm_run and wasm_run_flutter — they downloaded a 22 MB
binary from GitHub at first launch, violating the no-network-on-
default-launch-path policy.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-04-22 22:08:00 +02:00
co-authored by Claude
parent 4f5d97ebc5
commit ce8529e424
111 changed files with 7046 additions and 9 deletions
+8 -2
View File
@@ -2,11 +2,13 @@ import 'dart:async';
import 'package:clide/clide.dart';
import 'package:clide_app/kernel/kernel.dart';
import 'package:clide_app/kernel/src/syntax/tree_sitter_service.dart';
import 'package:clide_app/widgets/widgets.dart';
import 'package:flutter/services.dart';
import 'package:flutter/widgets.dart';
import 'editor_controller.dart';
import 'syntax_text_controller.dart';
/// Tier-2 editor tab. One tab — the content reflects the daemon's
/// active buffer. Multi-file tabs live in the workspace-slot plan but
@@ -25,14 +27,15 @@ class EditorView extends StatefulWidget {
class _EditorViewState extends State<EditorView> {
EditorController? _controller;
late final TextEditingController _text;
final TreeSitterService _syntax = TreeSitterService();
late final SyntaxTextController _text;
late final FocusNode _focus;
String? _lastRemoteContent;
@override
void initState() {
super.initState();
_text = TextEditingController();
_text = SyntaxTextController(syntax: _syntax);
_focus = FocusNode();
_text.addListener(_onTextChanged);
}
@@ -54,11 +57,13 @@ class _EditorViewState extends State<EditorView> {
_focus.dispose();
_controller?.removeListener(_onControllerChanged);
_controller?.dispose();
_syntax.dispose();
super.dispose();
}
void _onControllerChanged() {
final c = _controller!;
_text.updatePath(c.activePath);
if (c.content != _lastRemoteContent) {
_lastRemoteContent = c.content;
final sel = TextSelection(
@@ -110,6 +115,7 @@ class _EditorViewState extends State<EditorView> {
Widget build(BuildContext context) {
final c = _controller;
final tokens = ClideTheme.of(context).surface;
_text.tokens = tokens;
if (c == null) return const SizedBox.shrink();
return ListenableBuilder(
@@ -0,0 +1,161 @@
library;
import 'dart:convert';
import 'package:clide_app/kernel/src/syntax/language_map.dart';
import 'package:clide_app/kernel/src/syntax/tree_sitter_service.dart';
import 'package:clide_app/kernel/src/theme/tokens.dart';
import 'package:flutter/widgets.dart';
class SyntaxTextController extends TextEditingController {
SyntaxTextController({required TreeSitterService syntax})
: _syntax = syntax;
final TreeSitterService _syntax;
String? _highlightedPath;
String? _highlightedText;
List<SyntaxSpan> _spans = const [];
SurfaceTokens? _tokens;
bool _highlighting = false;
set tokens(SurfaceTokens value) => _tokens = value;
void updatePath(String? path) {
if (path == _highlightedPath) return;
_highlightedPath = path;
_spans = const [];
_highlightedText = null;
_requestHighlight();
}
void _requestHighlight() {
final path = _highlightedPath;
final source = text;
if (path == null || source.isEmpty || _highlighting) return;
if (grammarForPath(path) == null) return;
if (source == _highlightedText) return;
_highlighting = true;
_syntax.highlight(path, source).then((result) {
_highlighting = false;
if (text != source) {
_requestHighlight();
return;
}
_highlightedText = source;
_spans = result.spans;
notifyListeners();
}, onError: (_) {
_highlighting = false;
});
}
@override
set value(TextEditingValue newValue) {
super.value = newValue;
_requestHighlight();
}
@override
TextSpan buildTextSpan({
required BuildContext context,
TextStyle? style,
required bool withComposing,
}) {
final tokens = _tokens;
if (_spans.isEmpty || tokens == null || text.isEmpty) {
return TextSpan(text: text, style: style);
}
final source = text;
final sourceBytes = utf8.encode(source);
final children = <TextSpan>[];
// Convert byte offsets to character offsets.
// Build a byte-to-char map only up to the max byte we need.
final spans = _spans.where((s) => s.end <= sourceBytes.length).toList()
..sort((a, b) => a.start != b.start ? a.start - b.start : a.end - b.end);
if (spans.isEmpty) {
return TextSpan(text: text, style: style);
}
int maxByte = 0;
for (final s in spans) {
if (s.end > maxByte) maxByte = s.end;
}
// byte offset -> char offset lookup.
final byteToChar = List<int>.filled(maxByte + 1, 0);
int charIdx = 0;
int byteIdx = 0;
while (byteIdx <= maxByte && charIdx <= source.length) {
byteToChar[byteIdx] = charIdx;
if (charIdx < source.length) {
final codeUnit = source.codeUnitAt(charIdx);
// UTF-16 surrogate pair = 4 bytes in UTF-8.
if (codeUnit >= 0xD800 && codeUnit <= 0xDBFF) {
final bytesForPair = utf8.encode(source.substring(charIdx, charIdx + 2)).length;
for (var b = 1; b < bytesForPair && byteIdx + b <= maxByte; b++) {
byteToChar[byteIdx + b] = charIdx;
}
byteIdx += bytesForPair;
charIdx += 2;
} else {
final bytesForChar = utf8.encode(source[charIdx]).length;
for (var b = 1; b < bytesForChar && byteIdx + b <= maxByte; b++) {
byteToChar[byteIdx + b] = charIdx;
}
byteIdx += bytesForChar;
charIdx++;
}
} else {
break;
}
}
int charPos = 0;
for (final span in spans) {
if (span.start >= sourceBytes.length || span.end > sourceBytes.length) {
continue;
}
final spanCharStart = byteToChar[span.start];
final spanCharEnd = span.end <= maxByte
? byteToChar[span.end]
: source.length;
if (spanCharStart < charPos) continue;
// Gap before this span — plain text.
if (spanCharStart > charPos) {
children.add(TextSpan(
text: source.substring(charPos, spanCharStart),
style: style,
));
}
// The highlighted span.
if (spanCharEnd > spanCharStart) {
children.add(TextSpan(
text: source.substring(spanCharStart, spanCharEnd),
style: style?.copyWith(
color: TreeSitterService.colorForRole(span.role, tokens),
),
));
}
charPos = spanCharEnd;
}
// Trailing plain text.
if (charPos < source.length) {
children.add(TextSpan(
text: source.substring(charPos),
style: style,
));
}
return TextSpan(style: style, children: children);
}
}