add tree-sitter syntax highlighting via dart:ffi
Vendor libtree-sitter.so (v0.26.8, wasmtime 44.0 statically linked) and call its C API through dart:ffi. Grammar WASM files are loaded by tree-sitter's embedded wasmtime engine via ts_wasm_store_load_language. 48 grammars compiled with tree-sitter build --wasm, 48 highlight queries (42 from upstream repos, 6 written in-house for comment, erlang, gdscript, gitignore, hcl, hlsl, swift). SQLite grammar replaces full SQL (974K-line parser exhausted RAM at compile time). Removes wasm_run and wasm_run_flutter — they downloaded a 22 MB binary from GitHub at first launch, violating the no-network-on- default-launch-path policy. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -2,11 +2,13 @@ import 'dart:async';
|
||||
|
||||
import 'package:clide/clide.dart';
|
||||
import 'package:clide_app/kernel/kernel.dart';
|
||||
import 'package:clide_app/kernel/src/syntax/tree_sitter_service.dart';
|
||||
import 'package:clide_app/widgets/widgets.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:flutter/widgets.dart';
|
||||
|
||||
import 'editor_controller.dart';
|
||||
import 'syntax_text_controller.dart';
|
||||
|
||||
/// Tier-2 editor tab. One tab — the content reflects the daemon's
|
||||
/// active buffer. Multi-file tabs live in the workspace-slot plan but
|
||||
@@ -25,14 +27,15 @@ class EditorView extends StatefulWidget {
|
||||
|
||||
class _EditorViewState extends State<EditorView> {
|
||||
EditorController? _controller;
|
||||
late final TextEditingController _text;
|
||||
final TreeSitterService _syntax = TreeSitterService();
|
||||
late final SyntaxTextController _text;
|
||||
late final FocusNode _focus;
|
||||
String? _lastRemoteContent;
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
super.initState();
|
||||
_text = TextEditingController();
|
||||
_text = SyntaxTextController(syntax: _syntax);
|
||||
_focus = FocusNode();
|
||||
_text.addListener(_onTextChanged);
|
||||
}
|
||||
@@ -54,11 +57,13 @@ class _EditorViewState extends State<EditorView> {
|
||||
_focus.dispose();
|
||||
_controller?.removeListener(_onControllerChanged);
|
||||
_controller?.dispose();
|
||||
_syntax.dispose();
|
||||
super.dispose();
|
||||
}
|
||||
|
||||
void _onControllerChanged() {
|
||||
final c = _controller!;
|
||||
_text.updatePath(c.activePath);
|
||||
if (c.content != _lastRemoteContent) {
|
||||
_lastRemoteContent = c.content;
|
||||
final sel = TextSelection(
|
||||
@@ -110,6 +115,7 @@ class _EditorViewState extends State<EditorView> {
|
||||
Widget build(BuildContext context) {
|
||||
final c = _controller;
|
||||
final tokens = ClideTheme.of(context).surface;
|
||||
_text.tokens = tokens;
|
||||
if (c == null) return const SizedBox.shrink();
|
||||
|
||||
return ListenableBuilder(
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:clide_app/kernel/src/syntax/language_map.dart';
|
||||
import 'package:clide_app/kernel/src/syntax/tree_sitter_service.dart';
|
||||
import 'package:clide_app/kernel/src/theme/tokens.dart';
|
||||
import 'package:flutter/widgets.dart';
|
||||
|
||||
class SyntaxTextController extends TextEditingController {
|
||||
SyntaxTextController({required TreeSitterService syntax})
|
||||
: _syntax = syntax;
|
||||
|
||||
final TreeSitterService _syntax;
|
||||
|
||||
String? _highlightedPath;
|
||||
String? _highlightedText;
|
||||
List<SyntaxSpan> _spans = const [];
|
||||
SurfaceTokens? _tokens;
|
||||
bool _highlighting = false;
|
||||
|
||||
set tokens(SurfaceTokens value) => _tokens = value;
|
||||
|
||||
void updatePath(String? path) {
|
||||
if (path == _highlightedPath) return;
|
||||
_highlightedPath = path;
|
||||
_spans = const [];
|
||||
_highlightedText = null;
|
||||
_requestHighlight();
|
||||
}
|
||||
|
||||
void _requestHighlight() {
|
||||
final path = _highlightedPath;
|
||||
final source = text;
|
||||
if (path == null || source.isEmpty || _highlighting) return;
|
||||
if (grammarForPath(path) == null) return;
|
||||
if (source == _highlightedText) return;
|
||||
|
||||
_highlighting = true;
|
||||
_syntax.highlight(path, source).then((result) {
|
||||
_highlighting = false;
|
||||
if (text != source) {
|
||||
_requestHighlight();
|
||||
return;
|
||||
}
|
||||
_highlightedText = source;
|
||||
_spans = result.spans;
|
||||
notifyListeners();
|
||||
}, onError: (_) {
|
||||
_highlighting = false;
|
||||
});
|
||||
}
|
||||
|
||||
@override
|
||||
set value(TextEditingValue newValue) {
|
||||
super.value = newValue;
|
||||
_requestHighlight();
|
||||
}
|
||||
|
||||
@override
|
||||
TextSpan buildTextSpan({
|
||||
required BuildContext context,
|
||||
TextStyle? style,
|
||||
required bool withComposing,
|
||||
}) {
|
||||
final tokens = _tokens;
|
||||
if (_spans.isEmpty || tokens == null || text.isEmpty) {
|
||||
return TextSpan(text: text, style: style);
|
||||
}
|
||||
|
||||
final source = text;
|
||||
final sourceBytes = utf8.encode(source);
|
||||
final children = <TextSpan>[];
|
||||
|
||||
// Convert byte offsets to character offsets.
|
||||
// Build a byte-to-char map only up to the max byte we need.
|
||||
final spans = _spans.where((s) => s.end <= sourceBytes.length).toList()
|
||||
..sort((a, b) => a.start != b.start ? a.start - b.start : a.end - b.end);
|
||||
|
||||
if (spans.isEmpty) {
|
||||
return TextSpan(text: text, style: style);
|
||||
}
|
||||
|
||||
int maxByte = 0;
|
||||
for (final s in spans) {
|
||||
if (s.end > maxByte) maxByte = s.end;
|
||||
}
|
||||
|
||||
// byte offset -> char offset lookup.
|
||||
final byteToChar = List<int>.filled(maxByte + 1, 0);
|
||||
int charIdx = 0;
|
||||
int byteIdx = 0;
|
||||
while (byteIdx <= maxByte && charIdx <= source.length) {
|
||||
byteToChar[byteIdx] = charIdx;
|
||||
if (charIdx < source.length) {
|
||||
final codeUnit = source.codeUnitAt(charIdx);
|
||||
// UTF-16 surrogate pair = 4 bytes in UTF-8.
|
||||
if (codeUnit >= 0xD800 && codeUnit <= 0xDBFF) {
|
||||
final bytesForPair = utf8.encode(source.substring(charIdx, charIdx + 2)).length;
|
||||
for (var b = 1; b < bytesForPair && byteIdx + b <= maxByte; b++) {
|
||||
byteToChar[byteIdx + b] = charIdx;
|
||||
}
|
||||
byteIdx += bytesForPair;
|
||||
charIdx += 2;
|
||||
} else {
|
||||
final bytesForChar = utf8.encode(source[charIdx]).length;
|
||||
for (var b = 1; b < bytesForChar && byteIdx + b <= maxByte; b++) {
|
||||
byteToChar[byteIdx + b] = charIdx;
|
||||
}
|
||||
byteIdx += bytesForChar;
|
||||
charIdx++;
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int charPos = 0;
|
||||
for (final span in spans) {
|
||||
if (span.start >= sourceBytes.length || span.end > sourceBytes.length) {
|
||||
continue;
|
||||
}
|
||||
final spanCharStart = byteToChar[span.start];
|
||||
final spanCharEnd = span.end <= maxByte
|
||||
? byteToChar[span.end]
|
||||
: source.length;
|
||||
|
||||
if (spanCharStart < charPos) continue;
|
||||
|
||||
// Gap before this span — plain text.
|
||||
if (spanCharStart > charPos) {
|
||||
children.add(TextSpan(
|
||||
text: source.substring(charPos, spanCharStart),
|
||||
style: style,
|
||||
));
|
||||
}
|
||||
|
||||
// The highlighted span.
|
||||
if (spanCharEnd > spanCharStart) {
|
||||
children.add(TextSpan(
|
||||
text: source.substring(spanCharStart, spanCharEnd),
|
||||
style: style?.copyWith(
|
||||
color: TreeSitterService.colorForRole(span.role, tokens),
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
charPos = spanCharEnd;
|
||||
}
|
||||
|
||||
// Trailing plain text.
|
||||
if (charPos < source.length) {
|
||||
children.add(TextSpan(
|
||||
text: source.substring(charPos),
|
||||
style: style,
|
||||
));
|
||||
}
|
||||
|
||||
return TextSpan(style: style, children: children);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/// Maps file extensions and special filenames to grammar asset names.
|
||||
library;
|
||||
|
||||
String? grammarForPath(String path) {
|
||||
final name = path.split('/').last;
|
||||
final special = _filenameMap[name];
|
||||
if (special != null) return special;
|
||||
|
||||
final dot = name.lastIndexOf('.');
|
||||
if (dot < 0) return null;
|
||||
final ext = name.substring(dot).toLowerCase();
|
||||
return _extMap[ext];
|
||||
}
|
||||
|
||||
const _filenameMap = <String, String>{
|
||||
'Makefile': 'make',
|
||||
'makefile': 'make',
|
||||
'GNUmakefile': 'make',
|
||||
'Dockerfile': 'dockerfile',
|
||||
'dockerfile': 'dockerfile',
|
||||
'.gitignore': 'gitignore',
|
||||
'.gitconfig': 'git-config',
|
||||
'.gitmodules': 'git-config',
|
||||
'justfile': 'just',
|
||||
'Justfile': 'just',
|
||||
};
|
||||
|
||||
const _extMap = <String, String>{
|
||||
// Systems
|
||||
'.c': 'c',
|
||||
'.h': 'c',
|
||||
'.cpp': 'cpp',
|
||||
'.cxx': 'cpp',
|
||||
'.cc': 'cpp',
|
||||
'.hpp': 'cpp',
|
||||
'.hxx': 'cpp',
|
||||
'.cs': 'c-sharp',
|
||||
|
||||
// Application
|
||||
'.dart': 'dart',
|
||||
'.go': 'go',
|
||||
'.rs': 'rust',
|
||||
'.java': 'java',
|
||||
'.kt': 'kotlin',
|
||||
'.kts': 'kotlin',
|
||||
'.swift': 'swift',
|
||||
'.rb': 'ruby',
|
||||
'.py': 'python',
|
||||
'.pyw': 'python',
|
||||
'.ex': 'elixir',
|
||||
'.exs': 'elixir',
|
||||
'.erl': 'erlang',
|
||||
'.hrl': 'erlang',
|
||||
'.hs': 'haskell',
|
||||
'.lhs': 'haskell',
|
||||
'.jl': 'julia',
|
||||
'.r': 'r',
|
||||
'.R': 'r',
|
||||
'.zig': 'zig',
|
||||
'.nix': 'nix',
|
||||
'.lua': 'lua',
|
||||
'.php': 'php',
|
||||
'.nkl': 'nickel',
|
||||
|
||||
// Web
|
||||
'.js': 'javascript',
|
||||
'.mjs': 'javascript',
|
||||
'.cjs': 'javascript',
|
||||
'.jsx': 'javascript',
|
||||
'.ts': 'typescript',
|
||||
'.tsx': 'typescript',
|
||||
'.html': 'html',
|
||||
'.htm': 'html',
|
||||
'.css': 'css',
|
||||
'.svelte': 'svelte',
|
||||
'.vue': 'vue',
|
||||
|
||||
// Data / Config
|
||||
'.json': 'json',
|
||||
'.yaml': 'yaml',
|
||||
'.yml': 'yaml',
|
||||
'.toml': 'toml',
|
||||
'.xml': 'xml',
|
||||
'.svg': 'xml',
|
||||
'.plist': 'xml',
|
||||
'.hcl': 'hcl',
|
||||
'.tf': 'hcl',
|
||||
'.tfvars': 'hcl',
|
||||
'.proto': 'proto',
|
||||
'.regex': 'regex',
|
||||
|
||||
// Markdown
|
||||
'.md': 'markdown',
|
||||
'.mdx': 'markdown',
|
||||
'.markdown': 'markdown',
|
||||
|
||||
// Shell
|
||||
'.sh': 'bash',
|
||||
'.bash': 'bash',
|
||||
'.zsh': 'bash',
|
||||
'.fish': 'bash',
|
||||
|
||||
// Game dev
|
||||
'.gd': 'gdscript',
|
||||
'.glsl': 'glsl',
|
||||
'.vert': 'glsl',
|
||||
'.frag': 'glsl',
|
||||
'.hlsl': 'hlsl',
|
||||
'.wgsl': 'wgsl',
|
||||
|
||||
// Data / Query
|
||||
'.sql': 'sqlite',
|
||||
'.sqlite': 'sqlite',
|
||||
|
||||
// Other
|
||||
'.diff': 'diff',
|
||||
'.patch': 'diff',
|
||||
};
|
||||
@@ -0,0 +1,256 @@
|
||||
library;
|
||||
|
||||
import 'dart:ffi';
|
||||
import 'dart:io' show File, Platform;
|
||||
|
||||
import 'package:ffi/ffi.dart';
|
||||
|
||||
// -- Opaque handles ----------------------------------------------------------
|
||||
|
||||
final class TSParser extends Opaque {}
|
||||
final class TSTree extends Opaque {}
|
||||
final class TSQuery extends Opaque {}
|
||||
final class TSQueryCursor extends Opaque {}
|
||||
final class TSWasmStore extends Opaque {}
|
||||
final class TSWasmEngine extends Opaque {}
|
||||
|
||||
// -- Structs -----------------------------------------------------------------
|
||||
|
||||
final class TSNode extends Struct {
|
||||
@Array(4)
|
||||
external Array<Uint32> context;
|
||||
external Pointer<Void> id;
|
||||
external Pointer<Void> tree;
|
||||
}
|
||||
|
||||
final class TSQueryCapture extends Struct {
|
||||
external TSNode node;
|
||||
@Uint32()
|
||||
external int index;
|
||||
}
|
||||
|
||||
final class TSQueryMatch extends Struct {
|
||||
@Uint32()
|
||||
external int id;
|
||||
@Uint16()
|
||||
external int patternIndex;
|
||||
@Uint16()
|
||||
external int captureCount;
|
||||
external Pointer<TSQueryCapture> captures;
|
||||
}
|
||||
|
||||
final class TSWasmError extends Struct {
|
||||
@Int32()
|
||||
external int kind;
|
||||
external Pointer<Utf8> message;
|
||||
}
|
||||
|
||||
// -- Native function typedefs ------------------------------------------------
|
||||
|
||||
// Parser
|
||||
typedef _TsParserNew = Pointer<TSParser> Function();
|
||||
typedef _TsParserDelete = Void Function(Pointer<TSParser>);
|
||||
typedef _TsParserSetLanguage = Bool Function(Pointer<TSParser>, Pointer<Void>);
|
||||
typedef _TsParserSetWasmStore = Void Function(
|
||||
Pointer<TSParser>, Pointer<TSWasmStore>);
|
||||
typedef _TsParserParseString = Pointer<TSTree> Function(
|
||||
Pointer<TSParser>, Pointer<TSTree>, Pointer<Utf8>, Uint32);
|
||||
|
||||
// Tree
|
||||
typedef _TsTreeDelete = Void Function(Pointer<TSTree>);
|
||||
typedef _TsTreeRootNode = TSNode Function(Pointer<TSTree>);
|
||||
|
||||
// Node
|
||||
typedef _TsNodeStartByte = Uint32 Function(TSNode);
|
||||
typedef _TsNodeEndByte = Uint32 Function(TSNode);
|
||||
|
||||
// Query
|
||||
typedef _TsQueryNew = Pointer<TSQuery> Function(
|
||||
Pointer<Void>, Pointer<Utf8>, Uint32, Pointer<Uint32>, Pointer<Int32>);
|
||||
typedef _TsQueryDelete = Void Function(Pointer<TSQuery>);
|
||||
typedef _TsQueryCaptureCount = Uint32 Function(Pointer<TSQuery>);
|
||||
typedef _TsQueryCaptureNameForId = Pointer<Utf8> Function(
|
||||
Pointer<TSQuery>, Uint32, Pointer<Uint32>);
|
||||
|
||||
// Query cursor
|
||||
typedef _TsQueryCursorNew = Pointer<TSQueryCursor> Function();
|
||||
typedef _TsQueryCursorDelete = Void Function(Pointer<TSQueryCursor>);
|
||||
typedef _TsQueryCursorExec = Void Function(
|
||||
Pointer<TSQueryCursor>, Pointer<TSQuery>, TSNode);
|
||||
typedef _TsQueryCursorNextMatch = Bool Function(
|
||||
Pointer<TSQueryCursor>, Pointer<TSQueryMatch>);
|
||||
|
||||
// WASM store
|
||||
typedef _TsWasmStoreNew = Pointer<TSWasmStore> Function(
|
||||
Pointer<TSWasmEngine>, Pointer<TSWasmError>);
|
||||
typedef _TsWasmStoreDelete = Void Function(Pointer<TSWasmStore>);
|
||||
typedef _TsWasmStoreLoadLanguage = Pointer<Void> Function(
|
||||
Pointer<TSWasmStore>, Pointer<Utf8>, Pointer<Uint8>, Uint32,
|
||||
Pointer<TSWasmError>);
|
||||
|
||||
// WASM engine (from wasmtime C API, re-exported by tree-sitter)
|
||||
typedef _WasmEngineNew = Pointer<TSWasmEngine> Function();
|
||||
typedef _WasmEngineDelete = Void Function(Pointer<TSWasmEngine>);
|
||||
|
||||
// -- Dart function typedefs --------------------------------------------------
|
||||
|
||||
typedef DTsParserNew = Pointer<TSParser> Function();
|
||||
typedef DTsParserDelete = void Function(Pointer<TSParser>);
|
||||
typedef DTsParserSetLanguage = bool Function(Pointer<TSParser>, Pointer<Void>);
|
||||
typedef DTsParserSetWasmStore = void Function(
|
||||
Pointer<TSParser>, Pointer<TSWasmStore>);
|
||||
typedef DTsParserParseString = Pointer<TSTree> Function(
|
||||
Pointer<TSParser>, Pointer<TSTree>, Pointer<Utf8>, int);
|
||||
|
||||
typedef DTsTreeDelete = void Function(Pointer<TSTree>);
|
||||
typedef DTsTreeRootNode = TSNode Function(Pointer<TSTree>);
|
||||
|
||||
typedef DTsNodeStartByte = int Function(TSNode);
|
||||
typedef DTsNodeEndByte = int Function(TSNode);
|
||||
|
||||
typedef DTsQueryNew = Pointer<TSQuery> Function(
|
||||
Pointer<Void>, Pointer<Utf8>, int, Pointer<Uint32>, Pointer<Int32>);
|
||||
typedef DTsQueryDelete = void Function(Pointer<TSQuery>);
|
||||
typedef DTsQueryCaptureCount = int Function(Pointer<TSQuery>);
|
||||
typedef DTsQueryCaptureNameForId = Pointer<Utf8> Function(
|
||||
Pointer<TSQuery>, int, Pointer<Uint32>);
|
||||
|
||||
typedef DTsQueryCursorNew = Pointer<TSQueryCursor> Function();
|
||||
typedef DTsQueryCursorDelete = void Function(Pointer<TSQueryCursor>);
|
||||
typedef DTsQueryCursorExec = void Function(
|
||||
Pointer<TSQueryCursor>, Pointer<TSQuery>, TSNode);
|
||||
typedef DTsQueryCursorNextMatch = bool Function(
|
||||
Pointer<TSQueryCursor>, Pointer<TSQueryMatch>);
|
||||
|
||||
typedef DTsWasmStoreNew = Pointer<TSWasmStore> Function(
|
||||
Pointer<TSWasmEngine>, Pointer<TSWasmError>);
|
||||
typedef DTsWasmStoreDelete = void Function(Pointer<TSWasmStore>);
|
||||
typedef DTsWasmStoreLoadLanguage = Pointer<Void> Function(
|
||||
Pointer<TSWasmStore>, Pointer<Utf8>, Pointer<Uint8>, int,
|
||||
Pointer<TSWasmError>);
|
||||
|
||||
typedef DWasmEngineNew = Pointer<TSWasmEngine> Function();
|
||||
typedef DWasmEngineDelete = void Function(Pointer<TSWasmEngine>);
|
||||
|
||||
// -- Bindings ----------------------------------------------------------------
|
||||
|
||||
class TreeSitterLib {
|
||||
TreeSitterLib._(DynamicLibrary lib)
|
||||
: parserNew = lib.lookupFunction<_TsParserNew, DTsParserNew>(
|
||||
'ts_parser_new'),
|
||||
parserDelete = lib.lookupFunction<_TsParserDelete, DTsParserDelete>(
|
||||
'ts_parser_delete'),
|
||||
parserSetLanguage =
|
||||
lib.lookupFunction<_TsParserSetLanguage, DTsParserSetLanguage>(
|
||||
'ts_parser_set_language'),
|
||||
parserSetWasmStore =
|
||||
lib.lookupFunction<_TsParserSetWasmStore, DTsParserSetWasmStore>(
|
||||
'ts_parser_set_wasm_store'),
|
||||
parserParseString =
|
||||
lib.lookupFunction<_TsParserParseString, DTsParserParseString>(
|
||||
'ts_parser_parse_string'),
|
||||
treeDelete = lib.lookupFunction<_TsTreeDelete, DTsTreeDelete>(
|
||||
'ts_tree_delete'),
|
||||
treeRootNode = lib.lookupFunction<_TsTreeRootNode, DTsTreeRootNode>(
|
||||
'ts_tree_root_node'),
|
||||
nodeStartByte = lib.lookupFunction<_TsNodeStartByte, DTsNodeStartByte>(
|
||||
'ts_node_start_byte'),
|
||||
nodeEndByte = lib.lookupFunction<_TsNodeEndByte, DTsNodeEndByte>(
|
||||
'ts_node_end_byte'),
|
||||
queryNew =
|
||||
lib.lookupFunction<_TsQueryNew, DTsQueryNew>('ts_query_new'),
|
||||
queryDelete = lib.lookupFunction<_TsQueryDelete, DTsQueryDelete>(
|
||||
'ts_query_delete'),
|
||||
queryCaptureCount =
|
||||
lib.lookupFunction<_TsQueryCaptureCount, DTsQueryCaptureCount>(
|
||||
'ts_query_capture_count'),
|
||||
queryCaptureNameForId = lib.lookupFunction<_TsQueryCaptureNameForId,
|
||||
DTsQueryCaptureNameForId>('ts_query_capture_name_for_id'),
|
||||
queryCursorNew =
|
||||
lib.lookupFunction<_TsQueryCursorNew, DTsQueryCursorNew>(
|
||||
'ts_query_cursor_new'),
|
||||
queryCursorDelete =
|
||||
lib.lookupFunction<_TsQueryCursorDelete, DTsQueryCursorDelete>(
|
||||
'ts_query_cursor_delete'),
|
||||
queryCursorExec =
|
||||
lib.lookupFunction<_TsQueryCursorExec, DTsQueryCursorExec>(
|
||||
'ts_query_cursor_exec'),
|
||||
queryCursorNextMatch =
|
||||
lib.lookupFunction<_TsQueryCursorNextMatch, DTsQueryCursorNextMatch>(
|
||||
'ts_query_cursor_next_match'),
|
||||
wasmStoreNew = lib.lookupFunction<_TsWasmStoreNew, DTsWasmStoreNew>(
|
||||
'ts_wasm_store_new'),
|
||||
wasmStoreDelete =
|
||||
lib.lookupFunction<_TsWasmStoreDelete, DTsWasmStoreDelete>(
|
||||
'ts_wasm_store_delete'),
|
||||
wasmStoreLoadLanguage = lib.lookupFunction<_TsWasmStoreLoadLanguage,
|
||||
DTsWasmStoreLoadLanguage>('ts_wasm_store_load_language'),
|
||||
wasmEngineNew = lib.lookupFunction<_WasmEngineNew, DWasmEngineNew>(
|
||||
'wasm_engine_new'),
|
||||
wasmEngineDelete =
|
||||
lib.lookupFunction<_WasmEngineDelete, DWasmEngineDelete>(
|
||||
'wasm_engine_delete');
|
||||
|
||||
final DTsParserNew parserNew;
|
||||
final DTsParserDelete parserDelete;
|
||||
final DTsParserSetLanguage parserSetLanguage;
|
||||
final DTsParserSetWasmStore parserSetWasmStore;
|
||||
final DTsParserParseString parserParseString;
|
||||
final DTsTreeDelete treeDelete;
|
||||
final DTsTreeRootNode treeRootNode;
|
||||
final DTsNodeStartByte nodeStartByte;
|
||||
final DTsNodeEndByte nodeEndByte;
|
||||
final DTsQueryNew queryNew;
|
||||
final DTsQueryDelete queryDelete;
|
||||
final DTsQueryCaptureCount queryCaptureCount;
|
||||
final DTsQueryCaptureNameForId queryCaptureNameForId;
|
||||
final DTsQueryCursorNew queryCursorNew;
|
||||
final DTsQueryCursorDelete queryCursorDelete;
|
||||
final DTsQueryCursorExec queryCursorExec;
|
||||
final DTsQueryCursorNextMatch queryCursorNextMatch;
|
||||
final DTsWasmStoreNew wasmStoreNew;
|
||||
final DTsWasmStoreDelete wasmStoreDelete;
|
||||
final DTsWasmStoreLoadLanguage wasmStoreLoadLanguage;
|
||||
final DWasmEngineNew wasmEngineNew;
|
||||
final DWasmEngineDelete wasmEngineDelete;
|
||||
|
||||
static TreeSitterLib? _instance;
|
||||
|
||||
static TreeSitterLib? get instance => _instance;
|
||||
|
||||
static bool init() {
|
||||
if (_instance != null) return true;
|
||||
final lib = _openLibrary();
|
||||
if (lib == null) return false;
|
||||
_instance = TreeSitterLib._(lib);
|
||||
return true;
|
||||
}
|
||||
|
||||
static DynamicLibrary? _openLibrary() {
|
||||
final libName = Platform.isLinux
|
||||
? 'libtree-sitter.so'
|
||||
: Platform.isMacOS
|
||||
? 'libtree-sitter.dylib'
|
||||
: Platform.isWindows
|
||||
? 'tree-sitter.dll'
|
||||
: null;
|
||||
if (libName == null) return null;
|
||||
|
||||
// Try standard dlopen path first (works when lib is in bundle/lib/).
|
||||
try {
|
||||
return DynamicLibrary.open(libName);
|
||||
} catch (_) {}
|
||||
|
||||
// Try next to executable.
|
||||
final exe = File(Platform.resolvedExecutable).parent.path;
|
||||
for (final dir in ['$exe/lib', exe]) {
|
||||
final path = '$dir/$libName';
|
||||
if (File(path).existsSync()) {
|
||||
try {
|
||||
return DynamicLibrary.open(path);
|
||||
} catch (_) {}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
library;
|
||||
|
||||
import 'dart:convert' show utf8;
|
||||
import 'dart:ffi';
|
||||
import 'dart:ui' show Color;
|
||||
|
||||
import 'package:clide_app/kernel/src/syntax/language_map.dart';
|
||||
import 'package:clide_app/kernel/src/syntax/tree_sitter_ffi.dart';
|
||||
import 'package:clide_app/kernel/src/theme/tokens.dart';
|
||||
import 'package:ffi/ffi.dart';
|
||||
import 'package:flutter/services.dart' show rootBundle;
|
||||
|
||||
class SyntaxSpan {
|
||||
const SyntaxSpan({
|
||||
required this.start,
|
||||
required this.end,
|
||||
required this.role,
|
||||
});
|
||||
|
||||
final int start;
|
||||
final int end;
|
||||
final String role;
|
||||
}
|
||||
|
||||
class SyntaxResult {
|
||||
const SyntaxResult(this.spans);
|
||||
final List<SyntaxSpan> spans;
|
||||
|
||||
static const empty = SyntaxResult([]);
|
||||
}
|
||||
|
||||
class _LoadedGrammar {
|
||||
_LoadedGrammar({
|
||||
required this.language,
|
||||
required this.query,
|
||||
required this.captureNames,
|
||||
});
|
||||
|
||||
final Pointer<Void> language;
|
||||
final Pointer<TSQuery> query;
|
||||
final List<String> captureNames;
|
||||
}
|
||||
|
||||
class TreeSitterService {
|
||||
final Map<String, _LoadedGrammar> _grammars = {};
|
||||
final Set<String> _unavailable = {};
|
||||
|
||||
Pointer<TSWasmStore>? _store;
|
||||
Pointer<TSParser>? _parser;
|
||||
Pointer<TSQueryCursor>? _cursor;
|
||||
|
||||
bool _initDone = false;
|
||||
|
||||
bool _init() {
|
||||
if (_initDone) return _parser != null;
|
||||
_initDone = true;
|
||||
|
||||
final lib = TreeSitterLib.instance;
|
||||
if (lib == null) return false;
|
||||
|
||||
final engine = lib.wasmEngineNew();
|
||||
if (engine == nullptr) return false;
|
||||
|
||||
final error = calloc<TSWasmError>();
|
||||
_store = lib.wasmStoreNew(engine, error);
|
||||
lib.wasmEngineDelete(engine);
|
||||
|
||||
if (_store == null || _store == nullptr) {
|
||||
calloc.free(error);
|
||||
return false;
|
||||
}
|
||||
calloc.free(error);
|
||||
|
||||
_parser = lib.parserNew();
|
||||
if (_parser == null || _parser == nullptr) return false;
|
||||
lib.parserSetWasmStore(_parser!, _store!);
|
||||
|
||||
_cursor = lib.queryCursorNew();
|
||||
return true;
|
||||
}
|
||||
|
||||
Future<_LoadedGrammar?> _loadGrammar(String language) async {
|
||||
if (_unavailable.contains(language)) return null;
|
||||
final cached = _grammars[language];
|
||||
if (cached != null) return cached;
|
||||
|
||||
if (!_init()) {
|
||||
_unavailable.add(language);
|
||||
return null;
|
||||
}
|
||||
|
||||
final lib = TreeSitterLib.instance!;
|
||||
|
||||
try {
|
||||
// Load grammar WASM bytes.
|
||||
final wasmData =
|
||||
await rootBundle.load('assets/grammars/$language.wasm');
|
||||
final wasmBytes = wasmData.buffer.asUint8List();
|
||||
|
||||
// Load into WASM store.
|
||||
final nameNative = language.toNativeUtf8();
|
||||
final wasmNative = calloc<Uint8>(wasmBytes.length);
|
||||
wasmNative.asTypedList(wasmBytes.length).setAll(0, wasmBytes);
|
||||
final error = calloc<TSWasmError>();
|
||||
|
||||
final lang = lib.wasmStoreLoadLanguage(
|
||||
_store!, nameNative.cast(), wasmNative, wasmBytes.length, error,
|
||||
);
|
||||
|
||||
calloc.free(wasmNative);
|
||||
calloc.free(nameNative);
|
||||
|
||||
if (lang == nullptr) {
|
||||
final msg = error.ref.message;
|
||||
if (msg != nullptr) calloc.free(msg);
|
||||
calloc.free(error);
|
||||
_unavailable.add(language);
|
||||
return null;
|
||||
}
|
||||
calloc.free(error);
|
||||
|
||||
// Load highlight query.
|
||||
String? querySource;
|
||||
try {
|
||||
querySource =
|
||||
await rootBundle.loadString('assets/queries/$language.scm');
|
||||
} catch (_) {}
|
||||
|
||||
Pointer<TSQuery> query = nullptr;
|
||||
List<String> captureNames = [];
|
||||
|
||||
if (querySource != null) {
|
||||
final queryNative = querySource.toNativeUtf8();
|
||||
final queryLen = utf8.encode(querySource).length;
|
||||
final errorOffset = calloc<Uint32>();
|
||||
final errorType = calloc<Int32>();
|
||||
|
||||
query = lib.queryNew(
|
||||
lang, queryNative.cast(), queryLen, errorOffset, errorType,
|
||||
);
|
||||
|
||||
calloc.free(queryNative);
|
||||
calloc.free(errorOffset);
|
||||
calloc.free(errorType);
|
||||
|
||||
if (query != nullptr) {
|
||||
final count = lib.queryCaptureCount(query);
|
||||
final lenOut = calloc<Uint32>();
|
||||
for (var i = 0; i < count; i++) {
|
||||
final namePtr = lib.queryCaptureNameForId(query, i, lenOut);
|
||||
final len = lenOut.value;
|
||||
captureNames.add(namePtr.cast<Utf8>().toDartString(length: len));
|
||||
}
|
||||
calloc.free(lenOut);
|
||||
}
|
||||
}
|
||||
|
||||
final grammar = _LoadedGrammar(
|
||||
language: lang,
|
||||
query: query,
|
||||
captureNames: captureNames,
|
||||
);
|
||||
_grammars[language] = grammar;
|
||||
return grammar;
|
||||
} catch (_) {
|
||||
_unavailable.add(language);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Future<bool> hasGrammar(String path) async {
|
||||
final lang = grammarForPath(path);
|
||||
if (lang == null) return false;
|
||||
return (await _loadGrammar(lang)) != null;
|
||||
}
|
||||
|
||||
Future<String?> languageFor(String path) async {
|
||||
final lang = grammarForPath(path);
|
||||
if (lang == null) return null;
|
||||
return (await _loadGrammar(lang)) != null ? lang : null;
|
||||
}
|
||||
|
||||
List<String> get loadedLanguages => _grammars.keys.toList();
|
||||
|
||||
Future<SyntaxResult> highlight(String path, String source) async {
|
||||
final lang = grammarForPath(path);
|
||||
if (lang == null) return SyntaxResult.empty;
|
||||
|
||||
final grammar = await _loadGrammar(lang);
|
||||
if (grammar == null || grammar.query == nullptr) {
|
||||
return SyntaxResult.empty;
|
||||
}
|
||||
|
||||
final lib = TreeSitterLib.instance!;
|
||||
final parser = _parser!;
|
||||
final cursor = _cursor!;
|
||||
|
||||
// Set language on parser for this parse.
|
||||
lib.parserSetLanguage(parser, grammar.language);
|
||||
|
||||
// Parse source.
|
||||
final sourceNative = source.toNativeUtf8();
|
||||
final sourceLen = utf8.encode(source).length;
|
||||
final tree = lib.parserParseString(
|
||||
parser, nullptr, sourceNative.cast(), sourceLen,
|
||||
);
|
||||
|
||||
if (tree == nullptr) {
|
||||
calloc.free(sourceNative);
|
||||
return SyntaxResult.empty;
|
||||
}
|
||||
|
||||
final root = lib.treeRootNode(tree);
|
||||
|
||||
// Run highlight query.
|
||||
lib.queryCursorExec(cursor, grammar.query, root);
|
||||
|
||||
final match = calloc<TSQueryMatch>();
|
||||
final spans = <SyntaxSpan>[];
|
||||
|
||||
while (lib.queryCursorNextMatch(cursor, match)) {
|
||||
final m = match.ref;
|
||||
for (var i = 0; i < m.captureCount; i++) {
|
||||
final cap = m.captures[i];
|
||||
final captureIndex = cap.index;
|
||||
if (captureIndex < grammar.captureNames.length) {
|
||||
spans.add(SyntaxSpan(
|
||||
start: lib.nodeStartByte(cap.node),
|
||||
end: lib.nodeEndByte(cap.node),
|
||||
role: grammar.captureNames[captureIndex],
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
calloc.free(match);
|
||||
lib.treeDelete(tree);
|
||||
calloc.free(sourceNative);
|
||||
|
||||
return SyntaxResult(spans);
|
||||
}
|
||||
|
||||
void dispose() {
|
||||
final lib = TreeSitterLib.instance;
|
||||
if (lib == null) return;
|
||||
|
||||
for (final grammar in _grammars.values) {
|
||||
if (grammar.query != nullptr) lib.queryDelete(grammar.query);
|
||||
}
|
||||
_grammars.clear();
|
||||
|
||||
if (_cursor != null && _cursor != nullptr) lib.queryCursorDelete(_cursor!);
|
||||
// Parser and WASM store are cleaned up together — deleting the parser
|
||||
// does not delete the store, but the store owns the languages.
|
||||
if (_parser != null && _parser != nullptr) lib.parserDelete(_parser!);
|
||||
if (_store != null && _store != nullptr) lib.wasmStoreDelete(_store!);
|
||||
|
||||
_parser = null;
|
||||
_store = null;
|
||||
_cursor = null;
|
||||
_unavailable.clear();
|
||||
}
|
||||
|
||||
static Color colorForRole(String role, SurfaceTokens tokens) {
|
||||
return switch (role) {
|
||||
'keyword' || 'repeat' || 'conditional' || 'include' ||
|
||||
'exception' || 'operator' =>
|
||||
tokens.syntaxKeyword,
|
||||
'type' || 'type.builtin' || 'constructor' => tokens.syntaxType,
|
||||
'string' || 'string.special' => tokens.syntaxString,
|
||||
'number' || 'float' || 'boolean' => tokens.syntaxNumber,
|
||||
'comment' => tokens.syntaxComment,
|
||||
'function' || 'function.builtin' || 'function.method' ||
|
||||
'method' =>
|
||||
tokens.syntaxMethod,
|
||||
'punctuation.bracket' || 'punctuation.delimiter' ||
|
||||
'punctuation.special' =>
|
||||
tokens.syntaxPunct,
|
||||
'variable' || 'variable.builtin' || 'variable.parameter' =>
|
||||
tokens.globalForeground,
|
||||
'property' || 'field' => tokens.syntaxMethod,
|
||||
'constant' || 'constant.builtin' => tokens.syntaxNumber,
|
||||
'tag' || 'attribute' => tokens.syntaxKeyword,
|
||||
'namespace' || 'module' => tokens.syntaxType,
|
||||
'text.title' => tokens.syntaxKeyword,
|
||||
'text.literal' || 'text.reference' || 'text.uri' => tokens.syntaxString,
|
||||
'text.emphasis' || 'text.strong' => tokens.syntaxType,
|
||||
_ => tokens.globalForeground,
|
||||
};
|
||||
}
|
||||
}
|
||||
+3
-6
@@ -12,7 +12,6 @@ import 'package:clide_app/builtin/git/git.dart';
|
||||
import 'package:clide_app/builtin/grammars_core/grammars_core.dart';
|
||||
import 'package:clide_app/builtin/graph/graph.dart';
|
||||
import 'package:clide_app/builtin/ipc_status/ipc_status.dart';
|
||||
import 'package:clide_app/builtin/jira/jira.dart';
|
||||
import 'package:clide_app/builtin/keybindings_ui/keybindings_ui.dart';
|
||||
import 'package:clide_app/builtin/markdown/markdown.dart';
|
||||
import 'package:clide_app/builtin/pql/pql.dart';
|
||||
@@ -26,6 +25,7 @@ import 'package:clide_app/builtin/welcome/welcome.dart';
|
||||
import 'dart:io' show Directory, Platform;
|
||||
|
||||
import 'package:clide_app/kernel/kernel.dart';
|
||||
import 'package:clide_app/kernel/src/syntax/tree_sitter_ffi.dart';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/services.dart' show rootBundle;
|
||||
import 'package:flutter/widgets.dart';
|
||||
@@ -33,12 +33,10 @@ import 'package:flutter/widgets.dart';
|
||||
Future<void> main() async {
|
||||
final binding = WidgetsFlutterBinding.ensureInitialized();
|
||||
|
||||
// Enable the semantics tree at boot so screen readers (Orca, VoiceOver,
|
||||
// NVDA) + the headless-browser automation harness both work out of the
|
||||
// box. Production release builds could gate this on a platform check,
|
||||
// but a11y-first means always-on.
|
||||
binding.ensureSemantics();
|
||||
|
||||
TreeSitterLib.init();
|
||||
|
||||
final appDir = await _resolveAppDir();
|
||||
final themes = await _loadBundledThemes();
|
||||
|
||||
@@ -73,7 +71,6 @@ Future<void> main() async {
|
||||
..register(PqlExtension())
|
||||
..register(TodosExtension())
|
||||
..register(ProblemsExtension())
|
||||
..register(JiraExtension())
|
||||
..register(CanvasExtension())
|
||||
..register(GraphExtension())
|
||||
..register(SettingsUiExtension())
|
||||
|
||||
Reference in New Issue
Block a user