diff --git a/internal/grammar/semantic_tokens.go b/internal/grammar/semantic_tokens.go new file mode 100644 index 00000000..a3e6f4c1 --- /dev/null +++ b/internal/grammar/semantic_tokens.go @@ -0,0 +1,55 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package grammar + +import ( + "context" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/server" +) + +var legendProvider = server.NewExtendableSemanticTokensLegendProvider() + +type GrammarTokenHighlightingStrategy struct{} + +func NewGrammarTokenHighlightingStrategy() server.TokenHighlightingStrategy { + return &GrammarTokenHighlightingStrategy{} +} + +func (s *GrammarTokenHighlightingStrategy) Highlight(ctx context.Context, token core.Token, accept server.TokenHighlightingStrategyAcceptor) { + switch token.Kind { + case Grammar_Name_ID: + accept(legendProvider.Namespace(), 0) + case Interface_Name_ID, + Interface_Extends_ID_0, + Interface_Extends_ID_1: + accept(legendProvider.Interface(), 0) + case ParserRule_Name_ID, + Token_Name_ID, + CompositeRule_Name_ID, + RuleCall_Rule_ID, + TokenGroup_Name_ID, + TokenGroup_TokenRefs_ID: + accept(legendProvider.Function(), 0) + case Field_Name_ID, + Assignment_Property_ID, + Action_Property_ID: + accept(legendProvider.Property(), 0) + case PrimitiveType_Type_bool, + PrimitiveType_Type_composite, + PrimitiveType_Type_string, + SimpleType_Type_ID, + ReferenceType_Type_ID, + CrossRef_Type_ID, + ParserRule_ReturnType_ID, + Action_Type_ID, + Action_current: + accept(legendProvider.Type(), 0) + case Token_Type_comment, + Token_Type_hidden: + accept(legendProvider.Modifier(), 0) + } +} diff --git a/internal/grammar/semantic_tokens_test.go b/internal/grammar/semantic_tokens_test.go new file mode 100644 index 00000000..a3dbbebb --- /dev/null +++ b/internal/grammar/semantic_tokens_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package grammar + +import ( + "testing" + + "typefox.dev/fastbelt/test" +) + +func TestSemanticTokensIntegration(t *testing.T) { + fixture := test.New(t, CreateServices()) + + grammarText := `<|comment:// Grammar for semantic token testing|> +grammar <|namespace:Test|>; + +interface <|interface:Expression|> {} +interface <|interface:BinaryExpression|> extends <|interface:Expression|> { + <|property:Left|> <|type:Expression|> + <|property:Operator|> <|type:string|> + <|property:Right|> <|type:Expression|> +} + +<|function:Addition|> returns <|type:Expression|>: + <|function:Primary|> + ({<|type:BinaryExpression|>.<|property:Left|>=<|type:current|>} + <|property:Operator|>=("+" | "-") <|property:Right|>=<|function:Primary|>)* +<|function:Primary|> returns <|type:Expression|>: + <|property:Operator|>=<|function:ID|> + +token <|function:ID|>: /[a-zA-Z_][a-zA-Z0-9_]*/; +<|modifier:hidden|> token <|function:WS|>: /[ \n\r\t]+/; +` + + doc := fixture.ParseURI(grammarText, "file:///semantic.fb") + doc.AssertNoParseErrors() + semanticTokens := doc.ExpectSemanticTokens() + semanticTokens. + Assert("namespace", legendProvider.Namespace(), 0). + Assert("interface", legendProvider.Interface(), 0). + Assert("function", legendProvider.Function(), 0). + Assert("property", legendProvider.Property(), 0). + Assert("type", legendProvider.Type(), 0). + Assert("modifier", legendProvider.Modifier(), 0). + Assert("comment", legendProvider.Comment(), 0) +} diff --git a/internal/grammar/services.go b/internal/grammar/services.go index 1bf43378..bbf56926 100644 --- a/internal/grammar/services.go +++ b/internal/grammar/services.go @@ -8,6 +8,7 @@ package grammar import ( "typefox.dev/fastbelt/linking" + "typefox.dev/fastbelt/server" "typefox.dev/fastbelt/textdoc" "typefox.dev/fastbelt/util/service" "typefox.dev/fastbelt/workspace" @@ -29,6 +30,10 @@ func SetupServices(sc *service.Container) { // Override the default scope provider service.Override[FastbeltScopeProvider](sc, newScopeProviderImpl(sc)) service.Override(sc, newImportedSymbolsProviderImpl(sc)) + + // Set a semantic token highlighting strategy + service.Put[server.SemanticTokensLegendProvider](sc, legendProvider) + service.Put(sc, server.NewTokenBasedSemanticTokensProvider(sc, NewGrammarTokenHighlightingStrategy())) } // CreateServices creates a service container for the grammar language to be used in the CLI and tests. diff --git a/server/semantic_tokens_builder.go b/server/semantic_tokens_builder.go new file mode 100644 index 00000000..3c3e6182 --- /dev/null +++ b/server/semantic_tokens_builder.go @@ -0,0 +1,147 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package server + +import ( + "unicode/utf16" + "unicode/utf8" + + core "typefox.dev/fastbelt" +) + +// SemanticTokensBuilder defines the interface for building semantic tokens data in the LSP format. +// It provides methods to push individual tokens and retrieve the final data slice. +// +// It is recommended to build the semantic tokens data by using the [TokenBasedSemanticTokensProvider] and its +// associated [TokenHighlightingStrategy] implementations. +type SemanticTokensBuilder interface { + // Data returns the final semantic tokens data slice in the LSP format. + // The LSP data slice is a flat array of uint32 values, where each token is represented by five consecutive values: + // (1) deltaLine: token line number, relative to the previous token, + // (2) deltaStart: token start character, relative to the previous token, + // (3) length: the length of the token, + // (4) tokenType: the token type index in the legend, and + // (5) tokenModifiers: the token modifiers bitset in the legend. + Data() []uint32 + // Push adds a new token to the semantic tokens data. + // Tokens should be pushed in the order they appear in the document, as the LSP + // data is based on the offset of the previous token. + // Due to this, this method is not thread-safe. + Push(textRange core.TextRange, tokenType, tokenModifiers uint32) +} + +// NewSemanticTokensBuilder creates a new instance of [SemanticTokensBuilder]. +func NewSemanticTokensBuilder(text string, tokenCount int) SemanticTokensBuilder { + return &semanticTokensBuilder{ + // Preallocate the data with a reasonable length + // Each token potentially contributes 5 uint32 values + // Multiline tokens can contribute more, but we can resize the slice if needed + data: make([]uint32, 0, tokenCount*5), + text: text, + } +} + +type semanticTokensBuilder struct { + // data is a slice of uint32 values representing the semantic tokens data in the LSP format. + // Each token is represented by five consecutive values: + // - deltaLine, token line number, relative to the previous token, + // - deltaStart, token start character, relative to the previous token, + // - length, the length of the token, + // - tokenType, the token type index, + // - tokenModifiers, the token modifiers bitset. + data []uint32 + text string + cursor int + prevLine int + prevChar int + currentLine int + currentChar int + // lineBreaks is reused across push calls to avoid per-token allocations + lineBreaks []int +} + +func (tokenData *semanticTokensBuilder) Data() []uint32 { + return tokenData.data +} + +func (tokenData *semanticTokensBuilder) Push(textRange core.TextRange, typeIndex, modifierIndex uint32) { + textLen := len(tokenData.text) + tokenStart := int(textRange.Start) + tokenEnd := int(textRange.End) + cursor := tokenData.cursor + currentLine := tokenData.currentLine + currentChar := tokenData.currentChar + startLine, startChar := 0, 0 + // Count line breaks within the token range as necessary + // We need to emit multiple tokens if the token spans multiple lines + lineBreaks := tokenData.lineBreaks[:0] + // Advance the cursor up to the end of the token + for tokenEnd > cursor { + if cursor >= textLen { + break + } else if cursor == tokenStart { + startLine = currentLine + startChar = currentChar + } + if c := tokenData.text[cursor]; c < utf8.RuneSelf { + // ASCII fast path: one byte, one UTF-16 code unit + if c == '\n' { + // Record the line break character position + if cursor >= tokenStart { + lineBreaks = append(lineBreaks, currentChar) + } + // New line, reset currentChar and increment currentLine + currentLine++ + currentChar = 0 + } else { + currentChar++ + } + cursor++ + continue + } + rune, size := utf8.DecodeRuneInString(tokenData.text[cursor:]) + // Advance column by the number of UTF-16 code units for the rune + // (newlines are ASCII, so this rune can never be one) + currentChar += utf16.RuneLen(rune) + // Advance cursor by the byte size of the rune + cursor += size + } + tokenData.lineBreaks = lineBreaks + lineDelta := uint32(startLine - tokenData.prevLine) + charDelta := uint32(startChar) + if lineDelta == 0 { + // If the token is on the same line as the previous token, calculate the character delta + charDelta -= uint32(tokenData.prevChar) + } + if len(lineBreaks) == 0 { + // Token is on a single line, emit it directly + length := uint32(currentChar - startChar) + tokenData.data = append(tokenData.data, lineDelta, charDelta, length, typeIndex, modifierIndex) + // Update the previous character position for the next token + tokenData.prevChar = startChar + } else { + // Token spans multiple lines, emit a token for each line segment + // First segment: from startChar to the first line break + length := uint32(lineBreaks[0] - startChar) + tokenData.data = append(tokenData.data, lineDelta, charDelta, length, typeIndex, modifierIndex) + // Subsequent segments: from each line break to the next line break + for i := 1; i < len(lineBreaks); i++ { + // always use the full length of the line + length = uint32(lineBreaks[i]) + // Note: lineDelta is always 1, since each segment is on a new line + // charDelta is always 0, since we are starting at the beginning of the line + tokenData.data = append(tokenData.data, 1, 0, length, typeIndex, modifierIndex) + } + // Last segment: from the start of the last line to the end of the token + length = uint32(currentChar) + tokenData.data = append(tokenData.data, 1, 0, length, typeIndex, modifierIndex) + tokenData.prevChar = 0 + } + // Update the data for the next token + tokenData.cursor = cursor + tokenData.prevLine = currentLine + tokenData.currentLine = currentLine + tokenData.currentChar = currentChar +} diff --git a/server/semantic_tokens_builder_test.go b/server/semantic_tokens_builder_test.go new file mode 100644 index 00000000..df14fe66 --- /dev/null +++ b/server/semantic_tokens_builder_test.go @@ -0,0 +1,114 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package server + +import ( + "slices" + "testing" + + core "typefox.dev/fastbelt" +) + +func TestLspTokenDataPush(t *testing.T) { + tests := []struct { + name string + text string + ranges []core.TextRange + expected []uint32 + }{ + { + name: "Single token at start", + text: "hello world", + ranges: []core.TextRange{core.NewTextRange(0, 5)}, + expected: []uint32{0, 0, 5, 1, 2}, + }, + { + name: "Two tokens on same line use char delta", + text: "hello world", + ranges: []core.TextRange{core.NewTextRange(0, 5), core.NewTextRange(6, 11)}, + expected: []uint32{ + 0, 0, 5, 1, 2, + 0, 6, 5, 1, 2, + }, + }, + { + name: "Token on next line resets char delta", + text: "hello\nworld", + ranges: []core.TextRange{core.NewTextRange(0, 5), core.NewTextRange(6, 11)}, + expected: []uint32{ + 0, 0, 5, 1, 2, + 1, 0, 5, 1, 2, + }, + }, + { + name: "Multi-line token emits one token per line", + text: "ab\ncdef\ngh", + ranges: []core.TextRange{core.NewTextRange(1, 9)}, + expected: []uint32{ + 0, 1, 1, 1, 2, // "b" on line 0 + 1, 0, 4, 1, 2, // "cdef" on line 1 + 1, 0, 1, 1, 2, // "g" on line 2 + }, + }, + { + name: "Token after multi-line token", + text: "ab\ncd ef", + ranges: []core.TextRange{core.NewTextRange(0, 5), core.NewTextRange(6, 8)}, + expected: []uint32{ + 0, 0, 2, 1, 2, + 1, 0, 2, 1, 2, + 0, 3, 2, 1, 2, + }, + }, + { + name: "Non-ASCII counts UTF-16 code units", + // "😀" is 4 bytes but 2 UTF-16 code units + text: "😀ab", + ranges: []core.TextRange{core.NewTextRange(4, 6)}, + expected: []uint32{ + 0, 2, 2, 1, 2, + }, + }, + { + name: "Range past end of text is clamped", + text: "ab", + ranges: []core.TextRange{core.NewTextRange(0, 10)}, + expected: []uint32{0, 0, 2, 1, 2}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + builder := NewSemanticTokensBuilder(tt.text, len(tt.ranges)) + for _, rng := range tt.ranges { + builder.Push(rng, 1, 2) + } + if !slices.Equal(builder.Data(), tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, builder.Data()) + } + }) + } +} + +func BenchmarkLspTokenDataPush(b *testing.B) { + // Build a document of 1000 lines with 4 tokens each + line := "foo bar baz qux\n" + text := "" + ranges := []core.TextRange{} + for range 1000 { + offset := len(text) + for start := 0; start < 15; start += 4 { + ranges = append(ranges, core.NewTextRange(offset+start, offset+start+3)) + } + text += line + } + + for b.Loop() { + builder := NewSemanticTokensBuilder(text, len(ranges)) + for _, rng := range ranges { + builder.Push(rng, 1, 2) + } + } +} diff --git a/server/semantic_tokens_legend.go b/server/semantic_tokens_legend.go new file mode 100644 index 00000000..ca9c4445 --- /dev/null +++ b/server/semantic_tokens_legend.go @@ -0,0 +1,249 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package server + +import ( + "slices" + "sync" + + "typefox.dev/lsp" +) + +// SemanticTokensLegendProvider provides the legend for semantic tokens LSP requests. +// Must be registered together with a [SemanticTokensProvider] in the service container +// to enable semantic tokens support for the language server. +type SemanticTokensLegendProvider interface { + Legend() lsp.SemanticTokensLegend +} + +// ExtendableSemanticTokensLegendProvider is a type of [SemanticTokensLegendProvider] that allows +// adding new token types and modifiers at runtime, extending the default legend. +// +// Note that this interface is useful beyond the ability to add new token types and modifiers. +// It also provides a convenient way to retrieve the index of the default token types and the bit +// values of default token modifiers, which can be used when implementing custom semantic token provider. +// +// // Declaring a new legend provider with custom token types and modifiers +// var legendProvider = server.NewExtendableSemanticTokensLegendProvider() +// // Returns the index of the new token type within the legend +// var extraType = legendProvider.AddType("extraTokenType") +// // Returns the bit value of the new token modifier within the legend +// var extraModifier = legendProvider.AddModifier("extraTokenModifier") +// // Register within the service container +// service.Put[server.SemanticTokensLegendProvider](sc, legendProvider) +type ExtendableSemanticTokensLegendProvider interface { + SemanticTokensLegendProvider + + // Type returns the index of the "type" token type within the legend. + Type() uint32 + // Class returns the index of the "class" token type within the legend. + Class() uint32 + // Enum returns the index of the "enum" token type within the legend. + Enum() uint32 + // Interface returns the index of the "interface" token type within the legend. + Interface() uint32 + // Struct returns the index of the "struct" token type within the legend. + Struct() uint32 + // TypeParameter returns the index of the "typeParameter" token type within the legend. + TypeParameter() uint32 + // Parameter returns the index of the "parameter" token type within the legend. + Parameter() uint32 + // Variable returns the index of the "variable" token type within the legend. + Variable() uint32 + // Property returns the index of the "property" token type within the legend. + Property() uint32 + // EnumMember returns the index of the "enumMember" token type within the legend. + EnumMember() uint32 + // Event returns the index of the "event" token type within the legend. + Event() uint32 + // Function returns the index of the "function" token type within the legend. + Function() uint32 + // Method returns the index of the "method" token type within the legend. + Method() uint32 + // Macro returns the index of the "macro" token type within the legend. + Macro() uint32 + // Keyword returns the index of the "keyword" token type within the legend. + Keyword() uint32 + // Modifier returns the index of the "modifier" token type within the legend. + Modifier() uint32 + // Comment returns the index of the "comment" token type within the legend. + Comment() uint32 + // String returns the index of the "string" token type within the legend. + String() uint32 + // Number returns the index of the "number" token type within the legend. + Number() uint32 + // Regexp returns the index of the "regexp" token type within the legend. + Regexp() uint32 + // Operator returns the index of the "operator" token type within the legend. + Operator() uint32 + // Decorator returns the index of the "decorator" token type within the legend. + Decorator() uint32 + // Label returns the index of the "label" token type within the legend. + Label() uint32 + // Namespace returns the index of the "namespace" token type within the legend. + Namespace() uint32 + + // ModDeclaration returns the bit value of the "declaration" token modifier within the legend. + ModDeclaration() uint32 + // ModDefinition returns the bit value of the "definition" token modifier within the legend. + ModDefinition() uint32 + // ModReadonly returns the bit value of the "readonly" token modifier within the legend. + ModReadonly() uint32 + // ModStatic returns the bit value of the "static" token modifier within the legend. + ModStatic() uint32 + // ModDeprecated returns the bit value of the "deprecated" token modifier within the legend. + ModDeprecated() uint32 + // ModAbstract returns the bit value of the "abstract" token modifier within the legend. + ModAbstract() uint32 + // ModAsync returns the bit value of the "async" token modifier within the legend. + ModAsync() uint32 + // ModModification returns the bit value of the "modification" token modifier within the legend. + ModModification() uint32 + // ModDocumentation returns the bit value of the "documentation" token modifier within the legend. + ModDocumentation() uint32 + // ModDefaultLibrary returns the bit value of the "defaultLibrary" token modifier within the legend. + ModDefaultLibrary() uint32 + + // AddType adds a new token type to the legend and returns its index. + AddType(name string) uint32 + // AddModifier adds a new token modifier to the legend and returns its bit value. + // Note that the legend can only have a maximum of 32 token modifiers, so adding a new modifier + // when the legend already has 32 modifiers will result in a panic. + AddModifier(name string) uint32 +} + +// NewExtendableSemanticTokensLegendProvider creates a new instance of [ExtendableSemanticTokensLegendProvider]. +func NewExtendableSemanticTokensLegendProvider() ExtendableSemanticTokensLegendProvider { + return &extendableSemanticTokensLegendProvider{} +} + +func add(name string, existing *[]string) uint32 { + index := uint32(len(*existing)) + *existing = append(*existing, name) + return index +} + +var defaultTokenTypes []string + +var _type = add(string(lsp.TypeType), &defaultTokenTypes) +var _class = add(string(lsp.ClassType), &defaultTokenTypes) +var _enum = add(string(lsp.EnumType), &defaultTokenTypes) +var _interface = add(string(lsp.InterfaceType), &defaultTokenTypes) +var _struct = add(string(lsp.StructType), &defaultTokenTypes) +var _typeParameter = add(string(lsp.TypeParameterType), &defaultTokenTypes) +var _parameter = add(string(lsp.ParameterType), &defaultTokenTypes) +var _variable = add(string(lsp.VariableType), &defaultTokenTypes) +var _property = add(string(lsp.PropertyType), &defaultTokenTypes) +var _enumMember = add(string(lsp.EnumMemberType), &defaultTokenTypes) +var _event = add(string(lsp.EventType), &defaultTokenTypes) +var _function = add(string(lsp.FunctionType), &defaultTokenTypes) +var _method = add(string(lsp.MethodType), &defaultTokenTypes) +var _macro = add(string(lsp.MacroType), &defaultTokenTypes) +var _keyword = add(string(lsp.KeywordType), &defaultTokenTypes) +var _modifier = add(string(lsp.ModifierType), &defaultTokenTypes) +var _comment = add(string(lsp.CommentType), &defaultTokenTypes) +var _string = add(string(lsp.StringType), &defaultTokenTypes) +var _number = add(string(lsp.NumberType), &defaultTokenTypes) +var _regexp = add(string(lsp.RegexpType), &defaultTokenTypes) +var _operator = add(string(lsp.OperatorType), &defaultTokenTypes) +var _decorator = add(string(lsp.DecoratorType), &defaultTokenTypes) +var _label = add(string(lsp.LabelType), &defaultTokenTypes) +var _namespace = add(string(lsp.NamespaceType), &defaultTokenTypes) + +var defaultTokenModifiers []string + +var _modDeclaration uint32 = 1 << add(string(lsp.ModDeclaration), &defaultTokenModifiers) +var _modDefinition uint32 = 1 << add(string(lsp.ModDefinition), &defaultTokenModifiers) +var _modReadonly uint32 = 1 << add(string(lsp.ModReadonly), &defaultTokenModifiers) +var _modStatic uint32 = 1 << add(string(lsp.ModStatic), &defaultTokenModifiers) +var _modDeprecated uint32 = 1 << add(string(lsp.ModDeprecated), &defaultTokenModifiers) +var _modAbstract uint32 = 1 << add(string(lsp.ModAbstract), &defaultTokenModifiers) +var _modAsync uint32 = 1 << add(string(lsp.ModAsync), &defaultTokenModifiers) +var _modModification uint32 = 1 << add(string(lsp.ModModification), &defaultTokenModifiers) +var _modDocumentation uint32 = 1 << add(string(lsp.ModDocumentation), &defaultTokenModifiers) +var _modDefaultLibrary uint32 = 1 << add(string(lsp.ModDefaultLibrary), &defaultTokenModifiers) + +type extendableSemanticTokensLegendProvider struct { + types []string + modifiers []string + mutex sync.Mutex +} + +func (d *extendableSemanticTokensLegendProvider) Type() uint32 { return _type } +func (d *extendableSemanticTokensLegendProvider) Class() uint32 { return _class } +func (d *extendableSemanticTokensLegendProvider) Enum() uint32 { return _enum } +func (d *extendableSemanticTokensLegendProvider) Interface() uint32 { return _interface } +func (d *extendableSemanticTokensLegendProvider) Struct() uint32 { return _struct } +func (d *extendableSemanticTokensLegendProvider) TypeParameter() uint32 { return _typeParameter } +func (d *extendableSemanticTokensLegendProvider) Parameter() uint32 { return _parameter } +func (d *extendableSemanticTokensLegendProvider) Variable() uint32 { return _variable } +func (d *extendableSemanticTokensLegendProvider) Property() uint32 { return _property } +func (d *extendableSemanticTokensLegendProvider) EnumMember() uint32 { return _enumMember } +func (d *extendableSemanticTokensLegendProvider) Event() uint32 { return _event } +func (d *extendableSemanticTokensLegendProvider) Function() uint32 { return _function } +func (d *extendableSemanticTokensLegendProvider) Method() uint32 { return _method } +func (d *extendableSemanticTokensLegendProvider) Macro() uint32 { return _macro } +func (d *extendableSemanticTokensLegendProvider) Keyword() uint32 { return _keyword } +func (d *extendableSemanticTokensLegendProvider) Modifier() uint32 { return _modifier } +func (d *extendableSemanticTokensLegendProvider) Comment() uint32 { return _comment } +func (d *extendableSemanticTokensLegendProvider) String() uint32 { return _string } +func (d *extendableSemanticTokensLegendProvider) Number() uint32 { return _number } +func (d *extendableSemanticTokensLegendProvider) Regexp() uint32 { return _regexp } +func (d *extendableSemanticTokensLegendProvider) Operator() uint32 { return _operator } +func (d *extendableSemanticTokensLegendProvider) Decorator() uint32 { return _decorator } +func (d *extendableSemanticTokensLegendProvider) Label() uint32 { return _label } +func (d *extendableSemanticTokensLegendProvider) Namespace() uint32 { return _namespace } + +func (d *extendableSemanticTokensLegendProvider) ModDeclaration() uint32 { return _modDeclaration } +func (d *extendableSemanticTokensLegendProvider) ModDefinition() uint32 { return _modDefinition } +func (d *extendableSemanticTokensLegendProvider) ModReadonly() uint32 { return _modReadonly } +func (d *extendableSemanticTokensLegendProvider) ModStatic() uint32 { return _modStatic } +func (d *extendableSemanticTokensLegendProvider) ModDeprecated() uint32 { return _modDeprecated } +func (d *extendableSemanticTokensLegendProvider) ModAbstract() uint32 { return _modAbstract } +func (d *extendableSemanticTokensLegendProvider) ModAsync() uint32 { return _modAsync } +func (d *extendableSemanticTokensLegendProvider) ModModification() uint32 { return _modModification } +func (d *extendableSemanticTokensLegendProvider) ModDocumentation() uint32 { return _modDocumentation } +func (d *extendableSemanticTokensLegendProvider) ModDefaultLibrary() uint32 { + return _modDefaultLibrary +} + +func (d *extendableSemanticTokensLegendProvider) AddType(name string) uint32 { + d.mutex.Lock() + defer d.mutex.Unlock() + if slices.Contains(defaultTokenTypes, name) { + panic("Cannot add a token type that already exists in the default legend: " + name) + } else if slices.Contains(d.types, name) { + panic("Cannot add a token type that already exists in the legend: " + name) + } + return add(name, &d.types) + uint32(len(defaultTokenTypes)) +} + +func (d *extendableSemanticTokensLegendProvider) AddModifier(name string) uint32 { + d.mutex.Lock() + defer d.mutex.Unlock() + if slices.Contains(defaultTokenModifiers, name) { + panic("Cannot add a token modifier that already exists in the default legend: " + name) + } else if slices.Contains(d.modifiers, name) { + panic("Cannot add a token modifier that already exists in the legend: " + name) + } else if (len(d.modifiers) + len(defaultTokenModifiers)) >= 32 { + panic("Cannot add a token modifier because the legend already has 32 modifiers") + } + return 1 << (add(name, &d.modifiers) + uint32(len(defaultTokenModifiers))) +} + +func (d *extendableSemanticTokensLegendProvider) Legend() lsp.SemanticTokensLegend { + d.mutex.Lock() + defer d.mutex.Unlock() + tokenTypes := make([]string, len(defaultTokenTypes)+len(d.types)) + copy(tokenTypes, defaultTokenTypes) + copy(tokenTypes[len(defaultTokenTypes):], d.types) + tokenModifiers := make([]string, len(defaultTokenModifiers)+len(d.modifiers)) + copy(tokenModifiers, defaultTokenModifiers) + copy(tokenModifiers[len(defaultTokenModifiers):], d.modifiers) + return lsp.SemanticTokensLegend{ + TokenTypes: tokenTypes, + TokenModifiers: tokenModifiers, + } +} diff --git a/server/semantic_tokens_provider.go b/server/semantic_tokens_provider.go new file mode 100644 index 00000000..63c1d357 --- /dev/null +++ b/server/semantic_tokens_provider.go @@ -0,0 +1,137 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package server + +import ( + "context" + "errors" + "slices" + "strconv" + "strings" + "sync" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/util/service" + "typefox.dev/fastbelt/workspace" + "typefox.dev/lsp" +) + +// SemanticTokensProvider defines the interface for handling semantic tokens requests in the LSP. +// Must be registered together with a [SemanticTokensLegendProvider] in the service container +// to enable semantic tokens support for the language server. +type SemanticTokensProvider interface { + HandleSemanticTokensFullRequest(ctx context.Context, params *lsp.SemanticTokensParams) (*lsp.SemanticTokens, error) +} + +// TokenHighlightingStrategyAcceptor is a function type used in the [TokenHighlightingStrategy]. +type TokenHighlightingStrategyAcceptor func(tokenType uint32, tokenModifier uint32) + +// TokenHighlightingStrategy defines the interface for strategies that determine how individual tokens +// are highlighted by the [TokenBasedSemanticTokensProvider]. +// +// Note that the "accept" function should only be called once per token. +// Calling it multiple times for the same token will result in an error being returned to the language client. +type TokenHighlightingStrategy interface { + Highlight(ctx context.Context, token core.Token, accept TokenHighlightingStrategyAcceptor) +} + +// CommentTokenHighlightingStrategy extends the [TokenHighlightingStrategy] interface to include a method for highlighting comment tokens. +// If a [TokenHighlightingStrategy] also implements this interface, the [TokenBasedSemanticTokensProvider] will use it to highlight +// comment tokens in addition to regular tokens. +// Otherwise, comment tokens will be highlighted using the comment token type from the legend with no modifiers. +type CommentTokenHighlightingStrategy interface { + TokenHighlightingStrategy + HighlightComment(ctx context.Context, commentToken core.Token, accept TokenHighlightingStrategyAcceptor) +} + +// TokenBasedSemanticTokensProvider is an implementation of [SemanticTokensProvider] that generates semantic tokens +// for each individual token in the document, using a provided [TokenHighlightingStrategy] to determine the highlighting for each token. +// It also generates semantic tokens for comments in the document, if the "comment" token type is present in the legend. +type TokenBasedSemanticTokensProvider struct { + sc *service.Container + strategy TokenHighlightingStrategy + commentTypeIndexFunc func() int // Lazily initialized index of the comment token type in the legend +} + +// NewTokenBasedSemanticTokensProvider creates a new instance of [TokenBasedSemanticTokensProvider] with the given [TokenHighlightingStrategy]. +func NewTokenBasedSemanticTokensProvider(sc *service.Container, strategy TokenHighlightingStrategy) SemanticTokensProvider { + return &TokenBasedSemanticTokensProvider{sc: sc, strategy: strategy, commentTypeIndexFunc: sync.OnceValue(func() int { + tokenTypes := service.MustGet[SemanticTokensLegendProvider](sc).Legend().TokenTypes + commentTypeIndex := slices.Index(tokenTypes, string(lsp.CommentType)) + return commentTypeIndex + })} +} + +func (p *TokenBasedSemanticTokensProvider) HandleSemanticTokensFullRequest(ctx context.Context, params *lsp.SemanticTokensParams) (*lsp.SemanticTokens, error) { + documentManager := service.MustGet[workspace.DocumentManager](p.sc) + uri := core.ParseURI(string(params.TextDocument.URI)) + doc := documentManager.Get(uri) + if doc == nil { + return nil, nil // Document not found + } + tokens := doc.Tokens + comments := doc.Comments + totalLen := len(tokens) + len(comments) + if totalLen == 0 { + return nil, nil // Document is empty, no tokens found + } + commentTypeIndex := p.commentTypeIndexFunc() + tokenBuilder := NewSemanticTokensBuilder(doc.TextDoc.Text(nil), totalLen) + highlightComment := func(commentToken core.Token) {} + if commentStrategy, ok := p.strategy.(CommentTokenHighlightingStrategy); ok { + // Adopter has supplied a comment highlighting strategy, use that one + highlightComment = func(commentToken core.Token) { + commentStrategy.HighlightComment(ctx, commentToken, func(tokenType uint32, tokenModifier uint32) { + tokenBuilder.Push(commentToken.Range, tokenType, tokenModifier) + }) + } + } else if commentTypeIndex >= 0 { + // Highlight comments using the comment token type from the legend with no modifiers + highlightComment = func(commentToken core.Token) { + tokenBuilder.Push(commentToken.Range, uint32(commentTypeIndex), 0) + } + } + var errorRanges []core.TextRange + commentIndex := 0 + for _, token := range tokens { + for commentIndex < len(comments) && + comments[commentIndex].Range.Start < token.Range.Start { + // Add all comments that precede the current token + highlightComment(comments[commentIndex]) + commentIndex++ + } + added := false + p.strategy.Highlight(ctx, token, func(tokenType uint32, tokenModifier uint32) { + if !added { + tokenBuilder.Push(token.Range, tokenType, tokenModifier) + added = true + } else { + errorRanges = append(errorRanges, token.Range) + } + }) + } + // Report any tokens that were highlighted multiple times for the same range + if len(errorRanges) > 0 { + sb := strings.Builder{} + sb.WriteString("Multiple semantic tokens returned for the same token ranges: ") + for i, rng := range errorRanges { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(strconv.Itoa(int(rng.Start))) + sb.WriteString("-") + sb.WriteString(strconv.Itoa(int(rng.End))) + } + return nil, errors.New(sb.String()) + } + for commentIndex < len(comments) { + // Add remaining comments after the last token + highlightComment(comments[commentIndex]) + commentIndex++ + } + return &lsp.SemanticTokens{ + Data: tokenBuilder.Data(), + }, nil +} diff --git a/server/server.go b/server/server.go index b91b8881..15bf9cf8 100644 --- a/server/server.go +++ b/server/server.go @@ -6,6 +6,7 @@ package server import ( "context" + "errors" "log" "golang.org/x/exp/jsonrpc2" @@ -44,19 +45,33 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para return nil, err } workspaceFolders.Value = params.WorkspaceFolders - var triggerChars []string - if triggers, err := service.Get[CompletionTriggers](s.sc); err == nil && triggers != nil { - triggerChars = triggers.TriggerCharacters() + var completionOptions *lsp.CompletionOptions + if completionProvider, err := service.Get[CompletionProvider](s.sc); err == nil && completionProvider != nil { + completionOptions = &lsp.CompletionOptions{ + ResolveProvider: false, + } + if triggers, err := service.Get[CompletionTriggers](s.sc); err == nil && triggers != nil { + completionOptions.TriggerCharacters = triggers.TriggerCharacters() + } + } + var semanticTokensOptions *lsp.SemanticTokensOptions + if legendProvider, err := service.Get[SemanticTokensLegendProvider](s.sc); err == nil && legendProvider != nil { + semanticTokensOptions = &lsp.SemanticTokensOptions{ + Legend: legendProvider.Legend(), + Full: &lsp.Or_SemanticTokensOptions_full{ + Value: true, + }, + } + if !service.Has[SemanticTokensProvider](s.sc) { + return nil, errors.New("SemanticTokensLegendProvider is registered without a SemanticTokensProvider") + } } positionEncoding := lsp.UTF16 return &lsp.InitializeResult{ Capabilities: lsp.ServerCapabilities{ - PositionEncoding: &positionEncoding, - TextDocumentSync: lsp.Incremental, - CompletionProvider: &lsp.CompletionOptions{ - ResolveProvider: false, - TriggerCharacters: triggerChars, - }, + PositionEncoding: &positionEncoding, + TextDocumentSync: lsp.Incremental, + CompletionProvider: completionOptions, DefinitionProvider: &lsp.Or_ServerCapabilities_definitionProvider{ Value: service.Has[DefinitionProvider](s.sc), }, @@ -78,7 +93,8 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para ReferencesProvider: &lsp.Or_ServerCapabilities_referencesProvider{ Value: service.Has[ReferencesProvider](s.sc), }, - RenameProvider: service.Has[RenameProvider](s.sc), + RenameProvider: service.Has[RenameProvider](s.sc), + SemanticTokensProvider: semanticTokensOptions, }, }, nil } @@ -471,7 +487,22 @@ func (s *DefaultLanguageServer) SelectionRange(ctx context.Context, params *lsp. return nil, nil } func (s *DefaultLanguageServer) SemanticTokensFull(ctx context.Context, params *lsp.SemanticTokensParams) (*lsp.SemanticTokens, error) { - return nil, nil + lock, err := service.Get[workspace.Lock](s.sc) + if err != nil { + return nil, err + } + tokensProvider, err := service.Get[SemanticTokensProvider](s.sc) + if err != nil { + return nil, err + } + var result *lsp.SemanticTokens + var providerErr error + if err := lock.Read(ctx, func(ctx context.Context) { + result, providerErr = tokensProvider.HandleSemanticTokensFullRequest(ctx, params) + }); err != nil { + return nil, err + } + return result, providerErr } func (s *DefaultLanguageServer) SemanticTokensFullDelta(ctx context.Context, params *lsp.SemanticTokensDeltaParams) (any, error) { return nil, nil diff --git a/test/doc_fixture_lsp.go b/test/doc_fixture_lsp.go index a4de5511..dd3ab922 100644 --- a/test/doc_fixture_lsp.go +++ b/test/doc_fixture_lsp.go @@ -420,3 +420,90 @@ func findSymbolAtRange(symbols []lsp.DocumentSymbol, targetRange lsp.Range) *lsp } return nil } + +// ExpectSemanticTokens retrieves the semantic tokens for the document +// and returns a SemanticTokenExpectation for asserting on them. +func (d *Doc) ExpectSemanticTokens() *SemanticTokenExpectation { + d.fixture.t.Helper() + semanticTokensProvider := service.MustGet[server.SemanticTokensProvider](d.fixture.sc) + params := &lsp.SemanticTokensParams{ + TextDocument: lsp.TextDocumentIdentifier{ + URI: lsp.DocumentURI(d.Document.URI.DocumentURI()), + }, + } + result, err := semanticTokensProvider.HandleSemanticTokensFullRequest(d.fixture.ctx, params) + if err != nil { + d.fixture.t.Fatalf("fbtest: HandleSemanticTokensFullRequest returned error: %v", err) + } else if result == nil { + d.fixture.t.Fatalf("fbtest: HandleSemanticTokensFullRequest returned nil result") + } + var tokens []semanticToken + var line, column uint32 + for i := 0; i+4 < len(result.Data); i += 5 { + line += result.Data[i] + if result.Data[i] == 0 { + // Same line, column is relative to previous token + column += result.Data[i+1] + } else { + // New line, column is absolute + column = result.Data[i+1] + } + tokens = append(tokens, semanticToken{line, column, result.Data[i+2], result.Data[i+3], result.Data[i+4]}) + } + return &SemanticTokenExpectation{ + doc: d, + semanticTokens: tokens, + } +} + +// Decode the semantic tokens data, which comes in chunks of 5 uint32 values: +// [lineDelta, startCharDelta, length, tokenType, tokenModifiers] +type semanticToken struct { + line, column, length, tokenType, tokenModifiers uint32 +} + +// SemanticTokenExpectation is a helper struct for asserting semantic tokens in tests. +type SemanticTokenExpectation struct { + doc *Doc + semanticTokens []semanticToken +} + +// Assert verifies that every marker range with the given label +// has a semantic token with the expected type and modifiers. +// Returns the [SemanticTokenExpectation] for chaining. +func (e *SemanticTokenExpectation) Assert(label string, expectedType uint32, expectedModifiers uint32) *SemanticTokenExpectation { + d := e.doc + d.fixture.t.Helper() + ranges := d.markerRanges(label) + if len(ranges) == 0 { + d.fixture.t.Fatalf("fbtest: no marker with label %q", label) + } + for _, rng := range ranges { + startPosition := d.Document.TextDoc.PositionAt(int(rng.Start)) + endPosition := d.Document.TextDoc.PositionAt(int(rng.End)) + if startPosition.Line != endPosition.Line { + d.fixture.t.Fatalf("fbtest: AssertSemanticToken: marker %q spans multiple lines, which is not supported", label) + } + found := false + for _, token := range e.semanticTokens { + if token.line == startPosition.Line && token.column == startPosition.Character { + found = true + expectedLength := uint32(endPosition.Character - startPosition.Character) + if token.length != expectedLength { + d.fixture.t.Errorf("fbtest: semantic token at %q (%d:%d) has length %d, expected %d", label, token.line, token.column, token.length, expectedLength) + } + if token.tokenType != expectedType { + d.fixture.t.Errorf("fbtest: semantic token at %q (%d:%d) has type %d, expected %d", label, token.line, token.column, token.tokenType, expectedType) + } + if token.tokenModifiers != expectedModifiers { + d.fixture.t.Errorf("fbtest: semantic token at %q (%d:%d) has modifiers %d, expected %d", label, token.line, token.column, token.tokenModifiers, expectedModifiers) + } + break + } + } + if !found { + d.fixture.t.Errorf("fbtest: no semantic token found at %q (%d:%d)", label, startPosition.Line, startPosition.Character) + } + } + return e +}