Add semantic tokens support

internal/grammar/semantic_tokens.go

@@ -0,0 +1,55 @@
+// Copyright 2026 TypeFox GmbH
+// This program and the accompanying materials are made available under the
+// terms of the MIT License, which is available in the project root.
+
+package grammar
+
+import (
+	"context"
+
+	core "typefox.dev/fastbelt"
+	"typefox.dev/fastbelt/server"
+)
+
+var legendProvider = server.NewExtendableSemanticTokensLegendProvider()
+
+type GrammarTokenHighlightingStrategy struct{}
+
+func NewGrammarTokenHighlightingStrategy() server.TokenHighlightingStrategy {
+	return &GrammarTokenHighlightingStrategy{}
+}
+
+func (s *GrammarTokenHighlightingStrategy) Highlight(ctx context.Context, token core.Token, accept server.TokenHighlightingStrategyAcceptor) {
+	switch token.Kind {
+	case Grammar_Name_ID:
+		accept(legendProvider.Namespace(), 0)
+	case Interface_Name_ID,
+		Interface_Extends_ID_0,
+		Interface_Extends_ID_1:
+		accept(legendProvider.Interface(), 0)
+	case ParserRule_Name_ID,
+		Token_Name_ID,
+		CompositeRule_Name_ID,
+		RuleCall_Rule_ID,
+		TokenGroup_Name_ID,
+		TokenGroup_TokenRefs_ID:
+		accept(legendProvider.Function(), 0)
+	case Field_Name_ID,
+		Assignment_Property_ID,
+		Action_Property_ID:
+		accept(legendProvider.Property(), 0)
+	case PrimitiveType_Type_bool,
+		PrimitiveType_Type_composite,
+		PrimitiveType_Type_string,
+		SimpleType_Type_ID,
+		ReferenceType_Type_ID,
+		CrossRef_Type_ID,
+		ParserRule_ReturnType_ID,
+		Action_Type_ID,
+		Action_current:
+		accept(legendProvider.Type(), 0)
+	case Token_Type_comment,
+		Token_Type_hidden:
+		accept(legendProvider.Modifier(), 0)
+	}
+}

internal/grammar/semantic_tokens_test.go

@@ -0,0 +1,48 @@
+// Copyright 2026 TypeFox GmbH
+// This program and the accompanying materials are made available under the
+// terms of the MIT License, which is available in the project root.
+
+package grammar
+
+import (
+	"testing"
+
+	"typefox.dev/fastbelt/test"
+)
+
+func TestSemanticTokensIntegration(t *testing.T) {
+	fixture := test.New(t, CreateServices())
+
+	grammarText := `<|comment:// Grammar for semantic token testing|>
+grammar <|namespace:Test|>;
+
+interface <|interface:Expression|> {}
+interface <|interface:BinaryExpression|> extends <|interface:Expression|> {
+	<|property:Left|> <|type:Expression|>
+	<|property:Operator|> <|type:string|>
+	<|property:Right|> <|type:Expression|>
+}
+
+<|function:Addition|> returns <|type:Expression|>:
+	<|function:Primary|>
+	({<|type:BinaryExpression|>.<|property:Left|>=<|type:current|>}
+		<|property:Operator|>=("+" | "-") <|property:Right|>=<|function:Primary|>)*
+<|function:Primary|> returns <|type:Expression|>:
+	<|property:Operator|>=<|function:ID|>
+
+token <|function:ID|>: /[a-zA-Z_][a-zA-Z0-9_]*/;
+<|modifier:hidden|> token <|function:WS|>: /[ \n\r\t]+/;
+`
+
+	doc := fixture.ParseURI(grammarText, "file:///semantic.fb")
+	doc.AssertNoParseErrors()
+	semanticTokens := doc.ExpectSemanticTokens()
+	semanticTokens.
+		Assert("namespace", legendProvider.Namespace(), 0).
+		Assert("interface", legendProvider.Interface(), 0).
+		Assert("function", legendProvider.Function(), 0).
+		Assert("property", legendProvider.Property(), 0).
+		Assert("type", legendProvider.Type(), 0).
+		Assert("modifier", legendProvider.Modifier(), 0).
+		Assert("comment", legendProvider.Comment(), 0)
+}

internal/grammar/services.go

@@ -8,6 +8,7 @@ package grammar
 
 import (
 	"typefox.dev/fastbelt/linking"
+	"typefox.dev/fastbelt/server"
 	"typefox.dev/fastbelt/textdoc"
 	"typefox.dev/fastbelt/util/service"
 	"typefox.dev/fastbelt/workspace"
@@ -29,6 +30,10 @@ func SetupServices(sc *service.Container) {
 	// Override the default scope provider
 	service.Override[FastbeltScopeProvider](sc, newScopeProviderImpl(sc))
 	service.Override(sc, newImportedSymbolsProviderImpl(sc))
+
+	// Set a semantic token highlighting strategy
+	service.Put[server.SemanticTokensLegendProvider](sc, legendProvider)
+	service.Put(sc, server.NewTokenBasedSemanticTokensProvider(sc, NewGrammarTokenHighlightingStrategy()))
 }
 
 // CreateServices creates a service container for the grammar language to be used in the CLI and tests.

server/semantic_tokens_builder.go

@@ -0,0 +1,129 @@
+// Copyright 2026 TypeFox GmbH
+// This program and the accompanying materials are made available under the
+// terms of the MIT License, which is available in the project root.
+
+package server
+
+import (
+	"unicode/utf16"
+	"unicode/utf8"
+
+	core "typefox.dev/fastbelt"
+)
+
+type SemanticTokensBuilder interface {
+	Data() []uint32
+	Push(textRange core.TextRange, tokenType, tokenModifiers uint32)
+}
+
+func NewSemanticTokensBuilder(text string, tokenCount int) SemanticTokensBuilder {
+	return &semanticTokensBuilder{
+		// Preallocate the data with the maximum possible length
+		// Each token potentially contributes 5 uint32 values
+		data: make([]uint32, 0, tokenCount*5),
+		text: text,
+	}
+}
+
+type semanticTokensBuilder struct {
+	// data is a slice of uint32 values representing the semantic tokens data in the LSP format.
+	// Each token is represented by five consecutive values:
+	// - deltaLine, token line number, relative to the previous token,
+	// - deltaStart, token start character, relative to the previous token,
+	// - length, the length of the token,
+	// - tokenType, the token type index,
+	// - tokenModifiers, the token modifiers bitset.
+	data        []uint32
+	text        string
+	cursor      int
+	prevLine    int
+	prevChar    int
+	currentLine int
+	currentChar int
+	// lineBreaks is reused across push calls to avoid per-token allocations
+	lineBreaks []int
+}
+
+func (tokenData *semanticTokensBuilder) Data() []uint32 {
+	return tokenData.data
+}
+
+func (tokenData *semanticTokensBuilder) Push(textRange core.TextRange, typeIndex, modifierIndex uint32) {
+	textLen := len(tokenData.text)
+	tokenStart := int(textRange.Start)
+	tokenEnd := int(textRange.End)
+	cursor := tokenData.cursor
+	currentLine := tokenData.currentLine
+	currentChar := tokenData.currentChar
+	startLine, startChar := 0, 0
+	// Count line breaks within the token range as necessary
+	// We need to emit multiple tokens if the token spans multiple lines
+	lineBreaks := tokenData.lineBreaks[:0]
+	// Advance the cursor up to the end of the token
+	for tokenEnd > cursor {
+		if cursor >= textLen {
+			break
+		} else if cursor == tokenStart {
+			startLine = currentLine
+			startChar = currentChar
+		}
+		if c := tokenData.text[cursor]; c < utf8.RuneSelf {
+			// ASCII fast path: one byte, one UTF-16 code unit
+			if c == '\n' {
+				// Record the line break character position
+				if cursor >= tokenStart {
+					lineBreaks = append(lineBreaks, currentChar)
+				}
+				// New line, reset currentChar and increment currentLine
+				currentLine++
+				currentChar = 0
+			} else {
+				currentChar++
+			}
+			cursor++
+			continue
+		}
+		rune, size := utf8.DecodeRuneInString(tokenData.text[cursor:])
+		// Advance column by the number of UTF-16 code units for the rune
+		// (newlines are ASCII, so this rune can never be one)
+		currentChar += utf16.RuneLen(rune)
+		// Advance cursor by the byte size of the rune
+		cursor += size
+	}
+	tokenData.lineBreaks = lineBreaks
+	lineDelta := uint32(startLine - tokenData.prevLine)
+	charDelta := uint32(startChar)
+	if lineDelta == 0 {
+		// If the token is on the same line as the previous token, calculate the character delta
+		charDelta -= uint32(tokenData.prevChar)
+	}
+	if len(lineBreaks) == 0 {
+		// Token is on a single line, emit it directly
+		length := uint32(currentChar - startChar)
+		tokenData.data = append(tokenData.data, lineDelta, charDelta, length, typeIndex, modifierIndex)
+		// Update the previous character position for the next token
+		tokenData.prevChar = startChar
+	} else {
+		// Token spans multiple lines, emit a token for each line segment
+		// First segment: from startChar to the first line break
+		length := uint32(lineBreaks[0] - startChar)
+		tokenData.data = append(tokenData.data, lineDelta, charDelta, length, typeIndex, modifierIndex)
+		// Subsequent segments: from each line break to the next line break
+		for i := 1; i < len(lineBreaks); i++ {
+			// always use the full length of the line
+			length = uint32(lineBreaks[i])
+			// Note: lineDelta is always 1, since each segment is on a new line
+			// charDelta is always 0, since we are starting at the beginning of the line
+			tokenData.data = append(tokenData.data, 1, 0, length, typeIndex, modifierIndex)
+		}
+		// Last segment: from the start of the last line to the end of the token
+		length = uint32(currentChar)
+		tokenData.data = append(tokenData.data, 1, 0, length, typeIndex, modifierIndex)
+		tokenData.prevChar = 0
+	}
+	// Update the data for the next token
+	tokenData.cursor = cursor
+	tokenData.prevLine = currentLine
+	tokenData.currentLine = currentLine
+	tokenData.currentChar = currentChar
+}

server/semantic_tokens_builder_test.go

@@ -0,0 +1,114 @@
+// Copyright 2026 TypeFox GmbH
+// This program and the accompanying materials are made available under the
+// terms of the MIT License, which is available in the project root.
+
+package server
+
+import (
+	"slices"
+	"testing"
+
+	core "typefox.dev/fastbelt"
+)
+
+func TestLspTokenDataPush(t *testing.T) {
+	tests := []struct {
+		name     string
+		text     string
+		ranges   []core.TextRange
+		expected []uint32
+	}{
+		{
+			name:     "Single token at start",
+			text:     "hello world",
+			ranges:   []core.TextRange{core.NewTextRange(0, 5)},
+			expected: []uint32{0, 0, 5, 1, 2},
+		},
+		{
+			name:   "Two tokens on same line use char delta",
+			text:   "hello world",
+			ranges: []core.TextRange{core.NewTextRange(0, 5), core.NewTextRange(6, 11)},
+			expected: []uint32{
+				0, 0, 5, 1, 2,
+				0, 6, 5, 1, 2,
+			},
+		},
+		{
+			name:   "Token on next line resets char delta",
+			text:   "hello\nworld",
+			ranges: []core.TextRange{core.NewTextRange(0, 5), core.NewTextRange(6, 11)},
+			expected: []uint32{
+				0, 0, 5, 1, 2,
+				1, 0, 5, 1, 2,
+			},
+		},
+		{
+			name:   "Multi-line token emits one token per line",
+			text:   "ab\ncdef\ngh",
+			ranges: []core.TextRange{core.NewTextRange(1, 9)},
+			expected: []uint32{
+				0, 1, 1, 1, 2, // "b" on line 0
+				1, 0, 4, 1, 2, // "cdef" on line 1
+				1, 0, 1, 1, 2, // "g" on line 2
+			},
+		},
+		{
+			name:   "Token after multi-line token",
+			text:   "ab\ncd ef",
+			ranges: []core.TextRange{core.NewTextRange(0, 5), core.NewTextRange(6, 8)},
+			expected: []uint32{
+				0, 0, 2, 1, 2,
+				1, 0, 2, 1, 2,
+				0, 3, 2, 1, 2,
+			},
+		},
+		{
+			name: "Non-ASCII counts UTF-16 code units",
+			// "😀" is 4 bytes but 2 UTF-16 code units
+			text:   "😀ab",
+			ranges: []core.TextRange{core.NewTextRange(4, 6)},
+			expected: []uint32{
+				0, 2, 2, 1, 2,
+			},
+		},
+		{
+			name:     "Range past end of text is clamped",
+			text:     "ab",
+			ranges:   []core.TextRange{core.NewTextRange(0, 10)},
+			expected: []uint32{0, 0, 2, 1, 2},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			builder := NewSemanticTokensBuilder(tt.text, len(tt.ranges))
+			for _, rng := range tt.ranges {
+				builder.Push(rng, 1, 2)
+			}
+			if !slices.Equal(builder.Data(), tt.expected) {
+				t.Errorf("expected %v, got %v", tt.expected, builder.Data())
+			}
+		})
+	}
+}
+
+func BenchmarkLspTokenDataPush(b *testing.B) {
+	// Build a document of 1000 lines with 4 tokens each
+	line := "foo bar baz qux\n"
+	text := ""
+	ranges := []core.TextRange{}
+	for range 1000 {
+		offset := len(text)
+		for start := 0; start < 15; start += 4 {
+			ranges = append(ranges, core.NewTextRange(offset+start, offset+start+3))
+		}
+		text += line
+	}
+
+	for b.Loop() {
+		builder := NewSemanticTokensBuilder(text, len(ranges))
+		for _, rng := range ranges {
+			builder.Push(rng, 1, 2)
+		}
+	}
+}