diff --git a/.vscode/launch.json b/.vscode/launch.json index 578d184f..7b353b79 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -92,6 +92,34 @@ "outFiles": [ "${workspaceFolder}/internal/vscode-extensions/statemachine/dist/**/*.js" ] - } + }, + { + "name": "TokenModesWithGroups Generator", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceFolder}/cmd/fastbelt", + "cwd": "${workspaceFolder}/internal/languages/token_modes_with_groups", + "args": [ + "generate", + "token_modes_with_groups.fb", + "-v", + "--atn" + ] + }, + { + "name": "Completion Generator", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceFolder}/cmd/fastbelt", + "cwd": "${workspaceFolder}/internal/languages/completion", + "args": [ + "generate", + "completion.fb", + "-v", + "--atn" + ] + }, ] } \ No newline at end of file diff --git a/examples/arithmetics/benchmark_test.go b/examples/arithmetics/benchmark_test.go index 56c29503..9def258e 100644 --- a/examples/arithmetics/benchmark_test.go +++ b/examples/arithmetics/benchmark_test.go @@ -35,7 +35,7 @@ func BenchmarkParser(b *testing.B) { srv := CreateServices() lexerService := service.MustGet[lexer.Lexer](srv) parserService := service.MustGet[parser.Parser](srv) - lexed := lexerService.Lex(content) + lexed := lexerService.Exec(content) if len(lexed.Errors) > 0 { b.Fatalf("lexer errors: %v", lexed.Errors) } diff --git a/examples/arithmetics/lexer_gen.go b/examples/arithmetics/lexer_gen.go index a179c81b..30629c70 100644 --- a/examples/arithmetics/lexer_gen.go +++ b/examples/arithmetics/lexer_gen.go @@ -15,10 +15,7 @@ var Keyword_Percent = core.NewTokenType( Keyword_Percent_Idx, "%", "%", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "%") { return 1 @@ -34,10 +31,7 @@ var Keyword_LeftParen = core.NewTokenType( Keyword_LeftParen_Idx, "(", "(", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "(") { return 1 @@ -53,10 +47,7 @@ var Keyword_RightParen = core.NewTokenType( Keyword_RightParen_Idx, ")", ")", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ")") { return 1 @@ -72,10 +63,7 @@ var Keyword_Asterisk = core.NewTokenType( Keyword_Asterisk_Idx, "*", "*", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "*") { return 1 @@ -91,10 +79,7 @@ var Keyword_Plus = core.NewTokenType( Keyword_Plus_Idx, "+", "+", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "+") { return 1 @@ -110,10 +95,7 @@ var Keyword_Comma = core.NewTokenType( Keyword_Comma_Idx, ",", ",", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ",") { return 1 @@ -129,10 +111,7 @@ var Keyword_Dash = core.NewTokenType( Keyword_Dash_Idx, "-", "-", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "-") { return 1 @@ -148,10 +127,7 @@ var Keyword_Slash = core.NewTokenType( Keyword_Slash_Idx, "/", "/", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "/") { return 1 @@ -167,10 +143,7 @@ var Keyword_Colon = core.NewTokenType( Keyword_Colon_Idx, ":", ":", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ":") { return 1 @@ -186,10 +159,7 @@ var Keyword_Semicolon = core.NewTokenType( Keyword_Semicolon_Idx, ";", ";", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ";") { return 1 @@ -205,10 +175,7 @@ var Keyword_Caret = core.NewTokenType( Keyword_Caret_Idx, "^", "^", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "^") { return 1 @@ -224,10 +191,7 @@ var Keyword_def = core.NewTokenType( Keyword_def_Idx, "def", "def", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "def") { return 3 @@ -243,10 +207,7 @@ var Keyword_module = core.NewTokenType( Keyword_module_Idx, "module", "module", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "module") { return 6 @@ -262,10 +223,7 @@ var Token_ID = core.NewTokenType( Token_ID_Idx, "ID", "ID", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -336,10 +294,7 @@ var Token_NUMBER = core.NewTokenType( Token_NUMBER_Idx, "NUMBER", "NUMBER", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -428,10 +383,7 @@ var Token_WS = core.NewTokenType( Token_WS_Idx, "WS", "WS", - core.SkippedGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -502,10 +454,7 @@ var Token_ML_COMMENT = core.NewTokenType( Token_ML_COMMENT_Idx, "ML_COMMENT", "ML_COMMENT", - core.CommentGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -627,10 +576,7 @@ var Token_SL_COMMENT = core.NewTokenType( Token_SL_COMMENT_Idx, "SL_COMMENT", "SL_COMMENT", - core.SkippedGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -712,25 +658,31 @@ var Token_SL_COMMENT_Accepting = [3]bool{ 2: true, } +const ( + TokenMode_default = 0 +) + func NewLexer() lexer.Lexer { - return lexer.NewDefaultLexer( - Keyword_Percent, - Keyword_LeftParen, - Keyword_RightParen, - Keyword_Asterisk, - Keyword_Plus, - Keyword_Comma, - Keyword_Dash, - Keyword_Slash, - Keyword_Colon, - Keyword_Semicolon, - Keyword_Caret, - Keyword_def, - Keyword_module, - Token_ID, - Token_NUMBER, - Token_WS, - Token_ML_COMMENT, - Token_SL_COMMENT, + modes := make([]*lexer.TokenMode, 1) + modes[TokenMode_default] = lexer.NewTokenMode("default", + lexer.UseTokenType(Keyword_Percent), + lexer.UseTokenType(Keyword_LeftParen), + lexer.UseTokenType(Keyword_RightParen), + lexer.UseTokenType(Keyword_Asterisk), + lexer.UseTokenType(Keyword_Plus), + lexer.UseTokenType(Keyword_Comma), + lexer.UseTokenType(Keyword_Dash), + lexer.UseTokenType(Keyword_Slash), + lexer.UseTokenType(Keyword_Colon), + lexer.UseTokenType(Keyword_Semicolon), + lexer.UseTokenType(Keyword_Caret), + lexer.UseTokenType(Keyword_def), + lexer.UseTokenType(Keyword_module), + lexer.UseTokenType(Token_ID), + lexer.UseTokenType(Token_NUMBER), + lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier), + lexer.UseTokenType(Token_ML_COMMENT).WithModifier(core.CommentModifier), + lexer.UseTokenType(Token_SL_COMMENT).WithModifier(core.SkippedModifier), ) + return lexer.NewDefaultLexer(TokenMode_default, modes...) } diff --git a/examples/statemachine/benchmark_test.go b/examples/statemachine/benchmark_test.go index 5cdf559e..3dc00896 100644 --- a/examples/statemachine/benchmark_test.go +++ b/examples/statemachine/benchmark_test.go @@ -127,7 +127,7 @@ func BenchmarkParser(b *testing.B) { srv := CreateServices() lexerService := service.MustGet[lexer.Lexer](srv) parserService := service.MustGet[parser.Parser](srv) - tokens := lexerService.Lex(content).Tokens + tokens := lexerService.Exec(content).Tokens doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content) if err != nil { b.Fatal(err) @@ -148,7 +148,7 @@ func BenchmarkLexer(b *testing.B) { b.SetBytes(int64(len(content))) b.ResetTimer() for b.Loop() { - _ = l.Lex(content) + _ = l.Exec(content) } } @@ -165,7 +165,7 @@ func BenchmarkLexerAndParser(b *testing.B) { b.SetBytes(int64(len(content))) b.ResetTimer() for b.Loop() { - doc.Tokens = lexerService.Lex(content).Tokens + doc.Tokens = lexerService.Exec(content).Tokens result := parserService.Parse(doc) doc.Root = result.Node } diff --git a/examples/statemachine/lexer_gen.go b/examples/statemachine/lexer_gen.go index 6d237332..582bf695 100644 --- a/examples/statemachine/lexer_gen.go +++ b/examples/statemachine/lexer_gen.go @@ -15,10 +15,7 @@ var Keyword_EqualsGreaterThan = core.NewTokenType( Keyword_EqualsGreaterThan_Idx, "=>", "=>", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "=>") { return 2 @@ -34,10 +31,7 @@ var Keyword_actions = core.NewTokenType( Keyword_actions_Idx, "actions", "actions", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "actions") { return 7 @@ -53,10 +47,7 @@ var Keyword_commands = core.NewTokenType( Keyword_commands_Idx, "commands", "commands", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "commands") { return 8 @@ -72,10 +63,7 @@ var Keyword_end = core.NewTokenType( Keyword_end_Idx, "end", "end", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "end") { return 3 @@ -91,10 +79,7 @@ var Keyword_events = core.NewTokenType( Keyword_events_Idx, "events", "events", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "events") { return 6 @@ -110,10 +95,7 @@ var Keyword_initialState = core.NewTokenType( Keyword_initialState_Idx, "initialState", "initialState", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "initialState") { return 12 @@ -129,10 +111,7 @@ var Keyword_state = core.NewTokenType( Keyword_state_Idx, "state", "state", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "state") { return 5 @@ -148,10 +127,7 @@ var Keyword_statemachine = core.NewTokenType( Keyword_statemachine_Idx, "statemachine", "statemachine", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "statemachine") { return 12 @@ -167,10 +143,7 @@ var Keyword_LeftBrace = core.NewTokenType( Keyword_LeftBrace_Idx, "{", "{", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "{") { return 1 @@ -186,10 +159,7 @@ var Keyword_RightBrace = core.NewTokenType( Keyword_RightBrace_Idx, "}", "}", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "}") { return 1 @@ -205,10 +175,7 @@ var Token_ID = core.NewTokenType( Token_ID_Idx, "ID", "ID", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -279,10 +246,7 @@ var Token_WS = core.NewTokenType( Token_WS_Idx, "WS", "WS", - core.SkippedGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -353,10 +317,7 @@ var Token_ML_COMMENT = core.NewTokenType( Token_ML_COMMENT_Idx, "ML_COMMENT", "ML_COMMENT", - core.CommentGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -478,10 +439,7 @@ var Token_SL_COMMENT = core.NewTokenType( Token_SL_COMMENT_Idx, "SL_COMMENT", "SL_COMMENT", - core.CommentGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -563,21 +521,27 @@ var Token_SL_COMMENT_Accepting = [3]bool{ 2: true, } +const ( + TokenMode_default = 0 +) + func NewLexer() lexer.Lexer { - return lexer.NewDefaultLexer( - Keyword_EqualsGreaterThan, - Keyword_actions, - Keyword_commands, - Keyword_end, - Keyword_events, - Keyword_initialState, - Keyword_state, - Keyword_statemachine, - Keyword_LeftBrace, - Keyword_RightBrace, - Token_ID, - Token_WS, - Token_ML_COMMENT, - Token_SL_COMMENT, + modes := make([]*lexer.TokenMode, 1) + modes[TokenMode_default] = lexer.NewTokenMode("default", + lexer.UseTokenType(Keyword_EqualsGreaterThan), + lexer.UseTokenType(Keyword_actions), + lexer.UseTokenType(Keyword_commands), + lexer.UseTokenType(Keyword_end), + lexer.UseTokenType(Keyword_events), + lexer.UseTokenType(Keyword_initialState), + lexer.UseTokenType(Keyword_state), + lexer.UseTokenType(Keyword_statemachine), + lexer.UseTokenType(Keyword_LeftBrace), + lexer.UseTokenType(Keyword_RightBrace), + lexer.UseTokenType(Token_ID), + lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier), + lexer.UseTokenType(Token_ML_COMMENT).WithModifier(core.CommentModifier), + lexer.UseTokenType(Token_SL_COMMENT).WithModifier(core.CommentModifier), ) + return lexer.NewDefaultLexer(TokenMode_default, modes...) } diff --git a/examples/statemachine/statemachine.fb b/examples/statemachine/statemachine.fb index 1c798ebe..17042a56 100644 --- a/examples/statemachine/statemachine.fb +++ b/examples/statemachine/statemachine.fb @@ -58,4 +58,4 @@ token ID: /[_a-zA-Z][\w_]*/ hidden token WS: /\s+/ comment token ML_COMMENT: /\/\*[\s\S]*?\*\// -comment token SL_COMMENT: /\/\/[^\n\r]*/ +comment token SL_COMMENT: /\/\/[^\n\r]*/ \ No newline at end of file diff --git a/internal/atn/state_names.go b/internal/atn/state_names.go index c80b8678..1d832f23 100644 --- a/internal/atn/state_names.go +++ b/internal/atn/state_names.go @@ -53,7 +53,7 @@ func collectElementNames(names map[grammar.Element]string, prefix string, node f collectElementNames(names, prefix, n.Rule()) case grammar.RuleCall: ruleRef := n.Rule().Ref(context.Background()) - if token, ok := ruleRef.(grammar.Token); ok { + if token, ok := ruleRef.(grammar.TokenDecl); ok { names[n] = prefix + "_" + token.Name() } } diff --git a/internal/generator/atn_generator.go b/internal/generator/atn_generator.go index b6034dac..fc296cc2 100644 --- a/internal/generator/atn_generator.go +++ b/internal/generator/atn_generator.go @@ -10,7 +10,9 @@ import ( ) func GenerateATN(grammr grammar.Grammar, packageName string, tokenTypes GenerateTokenTypesResult) string { - a, _ := atn.CreateATN(grammr, tokenTypes.TokenTypeIds) - source := atn.EmitGoSource(packageName, a, grammr, tokenTypes.TokenTypeVarNames) + tokenTypeIds := tokenTypes.TokenTypeIds() + tokenTypeNames := tokenTypes.TokenTypeVarNamesByTokenIndex() + a, _ := atn.CreateATN(grammr, tokenTypeIds) + source := atn.EmitGoSource(packageName, a, grammr, tokenTypeNames) return FormatIfPossible(source.String()) } diff --git a/internal/generator/atn_md_generator.go b/internal/generator/atn_md_generator.go index 4595f538..9422b62f 100644 --- a/internal/generator/atn_md_generator.go +++ b/internal/generator/atn_md_generator.go @@ -10,7 +10,7 @@ import ( ) func GenerateATNMarkdown(grammr grammar.Grammar, packageName string, tokenTypes GenerateTokenTypesResult) string { - a, _ := atn.CreateATN(grammr, tokenTypes.TokenTypeIds) - source := atn.EmitMarkdownSource(packageName, grammr, a, tokenTypes.TokenTypeNames) + a, _ := atn.CreateATN(grammr, tokenTypes.TokenTypeIds()) + source := atn.EmitMarkdownSource(packageName, grammr, a, tokenTypes.TokenTypeNamesByTokenIndex()) return source.String() } diff --git a/internal/generator/lexer_generator.go b/internal/generator/lexer_generator.go index 6bef3650..9a008a7f 100644 --- a/internal/generator/lexer_generator.go +++ b/internal/generator/lexer_generator.go @@ -8,115 +8,25 @@ import ( "context" "fmt" "maps" - goregex "regexp" - "slices" "sort" "strconv" "unicode/utf8" "typefox.dev/fastbelt/internal/automatons" "typefox.dev/fastbelt/internal/grammar" - "typefox.dev/fastbelt/internal/regexp" + fbRegexp "typefox.dev/fastbelt/internal/regexp" "typefox.dev/fastbelt/util/codegen" ) -type GenerateTokenTypesResult struct { - Tokens []grammar.Token - Keywords []grammar.Keyword - Imports map[string]bool - KeywordsCode []codegen.Node - TokensCode []codegen.Node - TokenGroupCode []codegen.Node - TokenTypeVarNames []string - TokenTypeNames []string - TokenTypeIds map[string]int -} - -func GenerateTokenTypes(grammr grammar.Grammar) GenerateTokenTypesResult { - tokens := grammr.Terminals() - tokenGroups := grammr.TokenGroups() - keywords := GetAllKeywords(grammr) - keywordsCount := len(keywords) - result := GenerateTokenTypesResult{ - Tokens: tokens, - Keywords: keywords, - KeywordsCode: make([]codegen.Node, keywordsCount), - TokensCode: make([]codegen.Node, len(tokens)), - TokenGroupCode: make([]codegen.Node, len(tokenGroups)), - TokenTypeNames: make([]string, keywordsCount+len(tokens)+len(tokenGroups)), - TokenTypeVarNames: make([]string, keywordsCount+len(tokens)+len(tokenGroups)), - TokenTypeIds: make(map[string]int), - Imports: map[string]bool{}, - } - // Starting with 1 - prevent clash with EOF (index 0) - tokenIndex := 1 - for index, keyword := range keywords { - result.KeywordsCode[index] = generateKeywordTokenType(keyword, tokenIndex) - varName := GeneratedTokenName(keyword) - kwName := keyword.Value() - result.TokenTypeVarNames[index] = varName - result.TokenTypeNames[index] = kwName - result.TokenTypeIds[kwName] = index - result.Imports["strings"] = true - tokenIndex++ - } - for index, token := range tokens { - tokenType := generateTokenType(token, tokenIndex) - result.TokensCode[index] = tokenType.Code - for imp := range tokenType.Imports { - result.Imports[imp] = true - } - varName := GeneratedTokenName(token) - tokName := token.Name() - result.TokenTypeVarNames[keywordsCount+index] = varName - result.TokenTypeNames[keywordsCount+index] = tokName - result.TokenTypeIds[tokName] = keywordsCount + index - tokenIndex++ - } - tokenGroupMembers := map[string][]string{} - for _, tokenGroup := range tokenGroups { - tokenGroupMembers[tokenGroup.Name()] = getAllTokenGroupMembers(tokenGroup, keywords) - } - // Token groups need to be topologically sorted, so that nested groups appear after their members - for index, tokenGroup := range sortTokenGroups(tokenGroups, tokenGroupMembers) { - result.TokenGroupCode[index] = generateTokenGroupType(tokenGroup, tokenGroupMembers, tokenIndex) - varName := GeneratedTokenName(tokenGroup) - tokName := tokenGroup.Name() - result.TokenTypeVarNames[keywordsCount+len(tokens)+index] = varName - result.TokenTypeNames[keywordsCount+len(tokens)+index] = tokName - result.TokenTypeIds[tokName] = keywordsCount + len(tokens) + index - tokenIndex++ - } - return result -} - -func sortTokenGroups(tokenGroups []grammar.TokenGroup, members map[string][]string) []grammar.TokenGroup { - names := make([]string, len(tokenGroups)) - for i, tg := range tokenGroups { - names[i] = tg.Name() - } - sort.Strings(names) - topoSorted := topoSort(names, members, false) - sortedGroups := make([]grammar.TokenGroup, len(tokenGroups)) - for i, name := range topoSorted { - for _, tg := range tokenGroups { - if tg.Name() == name { - sortedGroups[i] = tg - break - } - } - } - return sortedGroups -} - func GenerateLexer(grammr grammar.Grammar, packageName string, tokenTypes GenerateTokenTypesResult) string { nodes := []codegen.Node{} imports := map[string]bool{} maps.Copy(imports, tokenTypes.Imports) - nodes = append(nodes, tokenTypes.KeywordsCode...) - nodes = append(nodes, tokenTypes.TokensCode...) - nodes = append(nodes, tokenTypes.TokenGroupCode...) + + for _, tokenType := range tokenTypes.TokenTypes.All { + nodes = append(nodes, tokenType.Code) + } node := NewRootNode() node.AppendLine("package ", packageName) @@ -142,28 +52,86 @@ func GenerateLexer(grammr grammar.Grammar, packageName string, tokenTypes Genera node.AppendLine() } - generateMainLexerFunction(node, tokenTypes.Tokens, tokenTypes.Keywords) + generateLexerModeEnums(node, tokenTypes) + generateMainLexerFunction(context.Background(), node, tokenTypes) return FormatIfPossible(node.String()) } -func generateMainLexerFunction(node codegen.Node, tokens []grammar.Token, keywords []grammar.Keyword) { +func generateLexerModeEnums(node codegen.Node, tokenTypes GenerateTokenTypesResult) { + node.AppendLine("const (") + node.Indent(func(n codegen.Node) { + for _, modeName := range tokenTypes.TokenModeOrder { + modeId := tokenTypes.TokenModes[modeName].Id + n.AppendLine("TokenMode_", modeName, " = ", strconv.Itoa(modeId)) + } + }) + node.AppendLine(")") + node.AppendLine() +} + +func generateMainLexerFunction(context context.Context, node codegen.Node, tokenTypes GenerateTokenTypesResult) { node.AppendLine("func NewLexer() lexer.Lexer {") node.Indent(func(n codegen.Node) { - n.AppendLine("return lexer.NewDefaultLexer(") - n.Indent(func(nn codegen.Node) { - for _, keyword := range keywords { - nn.AppendLine(GeneratedTokenName(keyword), ",") - } - for _, token := range tokens { - nn.AppendLine(GeneratedTokenName(token), ",") - } - }) - n.AppendLine(")") + count := strconv.Itoa(len(tokenTypes.TokenModes)) + n.AppendLine("modes := make([]*lexer.TokenMode, " + count + ")") + for _, modeName := range tokenTypes.TokenModeOrder { + tokenMode := tokenTypes.TokenModes[modeName] + varName := "modes[" + tokenTypes.TokenModes[modeName].VarName + "]" + n.AppendLine(varName, " = lexer.NewTokenMode(\"", modeName, "\",") + n.Indent(func(nn codegen.Node) { + for _, tokenIndex := range tokenMode.TokenTypeIndices.Keywords { + tokenType := tokenTypes.TokenTypes.ByTokenIndex[tokenIndex] + generateTokenTypeUsage(context, nn, tokenType, tokenMode, tokenIndex, tokenTypes) + } + for _, tokenIndex := range tokenMode.TokenTypeIndices.Tokens { + tokenType := tokenTypes.TokenTypes.ByTokenIndex[tokenIndex] + generateTokenTypeUsage(context, nn, tokenType, tokenMode, tokenIndex, tokenTypes) + } + }) + n.AppendLine(")") + } + n.AppendLine("return lexer.NewDefaultLexer(" + tokenTypes.TokenModes["default"].VarName + ", modes...)") }) node.AppendLine("}") } -func generateKeywordTokenType(keyword grammar.Keyword, id int) codegen.Node { +func generateTokenTypeUsage(context context.Context, nn codegen.Node, tokenType *TokenType, tokenMode *TokenMode, tokenIndex int, tokenTypes GenerateTokenTypesResult) { + nn.Append("lexer.UseTokenType(", tokenType.VarName, ")") + if usage, ok := tokenMode.TokenTypeUsages[tokenIndex]; ok { + if cmd := usage.Command; cmd != nil { + var cmdModeName string + if mode := cmd.Mode().Ref(context); cmd.IsDefault() || mode != nil { + if cmd.IsDefault() { + cmdModeName = "default" + } else { + cmdModeName = cmd.Mode().Ref(context).Name() + } + } + // A push/mode command without a resolvable target mode is reported + // by validation; skip it here instead of emitting a broken lexer. + targetMode := tokenTypes.TokenModes[cmdModeName] + switch { + case cmd.Type() == "pop": + nn.Append(".WithPopMode()") + case targetMode == nil: + // no target mode to switch to + case cmd.Type() == "push": + nn.Append(".WithPushMode(", targetMode.VarName, ")") + default: //"mode" + nn.Append(".WithSetMode(", targetMode.VarName, ")") + } + } + switch usage.TokenModifier { + case "comment": + nn.Append(".WithModifier(core.CommentModifier)") + case "hidden": + nn.Append(".WithModifier(core.SkippedModifier)") + } + } + nn.AppendLine(",") +} + +func generateKeywordTokenType(keyword grammar.Keyword, id int) GenerateLexerResult { code := codegen.NewNode() keywordValue := grammar.KeywordValue(keyword) code.AppendLine("const ", GeneratedTokenIdxName(keyword), " = ", strconv.Itoa(id)) @@ -173,10 +141,7 @@ func generateKeywordTokenType(keyword grammar.Keyword, id int) codegen.Node { n.AppendLine(GeneratedTokenIdxName(keyword), ",") n.AppendLine("\"", keywordValue, "\",") n.AppendLine("\"", keywordValue, "\",") - n.AppendLine("0,") n.AppendLine("core.TokenKindKeyword,") - n.AppendLine("0,") - n.AppendLine("false,") n.AppendLine("func (text string, offset int) int {") n.Indent(func(nn codegen.Node) { nn.AppendLine("if strings.HasPrefix(text[offset:], \"", keywordValue, "\") {") @@ -193,7 +158,12 @@ func generateKeywordTokenType(keyword grammar.Keyword, id int) codegen.Node { n.AppendLine("},") }) code.Append(")") - return code + return GenerateLexerResult{ + Imports: map[string]bool{ + "strings": true, + }, + Code: code, + } } type GenerateLexerResult struct { @@ -201,7 +171,7 @@ type GenerateLexerResult struct { Code codegen.Node } -func generateTokenGroupType(tokenGroup grammar.TokenGroup, tokenGroupMembers map[string][]string, id int) codegen.Node { +func generateTokenGroupType(tokenGroup grammar.TokenGroup, tokenGroupMembers map[string][]string, id int) GenerateLexerResult { code := codegen.NewNode() code.AppendLine("const ", GeneratedTokenIdxName(tokenGroup), " = ", strconv.Itoa(id)) code.AppendLine() @@ -217,49 +187,19 @@ func generateTokenGroupType(tokenGroup grammar.TokenGroup, tokenGroupMembers map n.AppendLine("},") }) code.Append(")") - return code -} - -func getAllTokenGroupMembers(tokenGroup grammar.TokenGroup, keywords []grammar.Keyword) []string { - members := map[string]bool{} - for _, tokenRef := range tokenGroup.TokenRefs() { - tokenRule := tokenRef.Ref(context.Background()) - if tokenRule != nil { - name := GeneratedTokenName(tokenRule) - members[name] = true - } - } - for _, keyword := range tokenGroup.Keywords() { - name := GeneratedTokenName(keyword) - members[name] = true - } - for _, regex := range tokenGroup.Regexps() { - pattern := regex.Image[1 : len(regex.Image)-1] - compiled, err := goregex.Compile(pattern) - if err != nil { - continue - } - for _, keyword := range keywords { - keywordValue := KeywordValue(keyword) - keywordName := GeneratedTokenName(keyword) - matches := compiled.MatchString(keywordValue) - if matches { - members[keywordName] = true - } - } + return GenerateLexerResult{ + Imports: map[string]bool{}, + Code: code, } - slice := slices.Collect(maps.Keys(members)) - sort.Strings(slice) - return slice } -func generateTokenType(token grammar.Token, id int) GenerateLexerResult { - var result regexp.GenerateRegExpResult +func generateRegexpTokenElement(token grammar.TokenDecl, regexpTokenElement grammar.RegexpTokenContent, id int) GenerateLexerResult { + var result fbRegexp.GenerateRegExpResult imports := map[string]bool{} code := codegen.NewNode() - regexPattern := token.Regexp() - regexPattern = regexPattern[1 : len(regexPattern)-1] // remove leading and trailing backticks - regex, err := regexp.Compile(regexPattern) + regexPattern := regexpTokenElement.Regexp() + regexPattern = grammar.RegexpValue(regexPattern) + regex, err := fbRegexp.Compile(regexPattern) if err != nil { panic(err) } @@ -269,17 +209,8 @@ func generateTokenType(token grammar.Token, id int) GenerateLexerResult { n.AppendLine(GeneratedTokenIdxName(token), ",") n.AppendLine("\"", token.Name(), "\",") n.AppendLine("\"", token.Name(), "\",") - if token.Type() == "hidden" { - n.AppendLine("core.SkippedGroup,") - } else if token.Type() == "comment" { - n.AppendLine("core.CommentGroup,") - } else { - n.AppendLine("0,") - } n.AppendLine("core.TokenKindToken,") - n.AppendLine("0,") - n.AppendLine("false,") - impl := regex.(*regexp.RegexpImpl) + impl := regex.(*fbRegexp.RegexpImpl) result = impl.GenerateRegExp("", GeneratedTokenName(token)) for imp := range result.Imports { imports[imp] = true diff --git a/internal/generator/lexer_generator_test.go b/internal/generator/lexer_generator_test.go new file mode 100644 index 00000000..445effef --- /dev/null +++ b/internal/generator/lexer_generator_test.go @@ -0,0 +1,354 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package generator + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "typefox.dev/fastbelt/internal/grammar" + "typefox.dev/fastbelt/test" +) + +// generateLexerFor generates the lexer for src without asserting that src is +// free of diagnostics, so malformed grammars can be exercised too. +func generateLexerFor(t *testing.T, src string) string { + t.Helper() + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(src) + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + return GenerateLexer(grammr, "test", GenerateTokenTypes(grammr)) +} + +func TestGenerateLexerModeCommands(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID -> push(Other) + hidden WS + } + token mode Other { + ID -> pop + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_ID).WithPushMode(TokenMode_Other)") + assert.Contains(t, code, "lexer.UseTokenType(Token_ID).WithPopMode()") + assert.Contains(t, code, "lexer.NewDefaultLexer(TokenMode_default, modes...)") +} + +func TestGenerateLexerSetModeCommand(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID -> push(Other) + hidden WS + } + token mode Other { + ID -> mode(default) + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_ID).WithSetMode(TokenMode_default)") +} + +// A push command without a target mode is a validation error. Code generation +// must still terminate instead of panicking, because GenerateLexer is reachable +// for grammars that have not been validated. +func TestGenerateLexerPushCommandWithoutTargetMode(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID -> push + hidden WS + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_ID)") + assert.NotContains(t, code, "WithPushMode") +} + +func TestGenerateLexerPushCommandWithUnknownTargetMode(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID -> push(Missing) + hidden WS + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_ID)") + assert.NotContains(t, code, "WithPushMode") +} + +// A pop command keeps working even when a target mode was given, which is what +// the corresponding validation error tells the user. +func TestGenerateLexerPopCommandIgnoresTargetMode(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID -> push(Other) + hidden WS + } + token mode Other { + ID -> pop(default) + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_ID).WithPopMode()") + assert.NotContains(t, code, "WithSetMode") +} + +func TestGenerateLexerPushDefaultModeCommand(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID -> push(Other) + hidden WS + } + token mode Other { + ID -> push(default) + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_ID).WithPushMode(TokenMode_default)") +} + +// --- Mode ordering and identifiers --- + +func TestGenerateLexerModeIdsFollowDeclarationOrder(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode First { + ID -> push(default) + } + token mode default { + ID -> push(First) + hidden WS + } + `) + // The default mode is declared second, so it gets id 1 and the lexer must + // still start in it. + assert.Contains(t, code, "TokenMode_First = 0") + assert.Contains(t, code, "TokenMode_default = 1") + assert.Contains(t, code, "modes := make([]*lexer.TokenMode, 2)") + assert.Contains(t, code, "lexer.NewDefaultLexer(TokenMode_default, modes...)") +} + +func TestGenerateLexerImplicitDefaultModeRegistersEverything(t *testing.T) { + // Without any token mode declaration the generator synthesizes a default + // mode from all top-level keywords, tokens and token groups. + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID "world" Punct; + token group Punct { "!" } + token ID: /[a-z]+/ + hidden token WS: /\s+/ + comment token SL_COMMENT: /\/\/[^\n]*/ + `) + assert.Contains(t, code, "TokenMode_default = 0") + assert.Contains(t, code, "lexer.UseTokenType(Keyword_world)") + assert.Contains(t, code, "lexer.UseTokenType(Token_ID)") + assert.Contains(t, code, "lexer.UseTokenType(Keyword_Exclamation),") + assert.Contains(t, code, "lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier)") + assert.Contains(t, code, "lexer.UseTokenType(Token_SL_COMMENT).WithModifier(core.CommentModifier)") + assert.Contains(t, code, "lexer.NewDefaultLexer(TokenMode_default, modes...)") +} + +// --- Modifier and command precedence --- + +func TestGenerateLexerModeUsageInheritsGlobalModifier(t *testing.T) { + // WS is hidden globally and listed without a modifier in the mode, so the + // global modifier carries over - `hidden` does not have to be repeated. + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID + WS + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier)") +} + +func TestGenerateLexerModeUsageOverridesGlobalModifier(t *testing.T) { + // A modifier on the mode entry replaces the global one. + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token NOTE: /\/\/[^\n]*/ + token mode default { + ID + comment NOTE + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_NOTE).WithModifier(core.CommentModifier)") + assert.NotContains(t, code, "Token_NOTE).WithModifier(core.SkippedModifier)") +} + +func TestGenerateLexerModeUsageAddsModifierToGlobalToken(t *testing.T) { + // The reverse direction: a plain global token is hidden inside the mode. + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + token WS: /\s+/ + token mode default { + ID + hidden WS + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier)") +} + +func TestGenerateLexerModeUsageInheritsGlobalCommentModifier(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + comment token SL_COMMENT: /\/\/[^\n]*/ + token mode default { + ID + SL_COMMENT + } + `) + assert.Contains(t, code, "lexer.UseTokenType(Token_SL_COMMENT).WithModifier(core.CommentModifier)") +} + +// --- Mode-local declarations --- + +func TestGenerateLexerModeLocalTokenDeclaration(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string Content string } + Foo: Greeting=ID "(" Content=INNER ")"; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID + "(" -> push(Inner) + hidden WS + } + token mode Inner { + token INNER: /[A-Z]+/ + ")" -> pop + } + `) + // The mode-local declaration produces its own token type and is registered + // only in the mode that declares it. + assert.Contains(t, code, "var Token_INNER =") + inner := code[strings.Index(code, `NewTokenMode("Inner"`):] + assert.Contains(t, inner, "lexer.UseTokenType(Token_INNER)") + outer := code[strings.Index(code, `NewTokenMode("default"`):strings.Index(code, `NewTokenMode("Inner"`)] + assert.NotContains(t, outer, "Token_INNER") +} + +// A mode-local declaration is visible to parser rules but not to a token usage +// in a different mode, which the linker reports. The generated lexer must still +// be well-formed - the unresolvable entry is dropped rather than emitted broken. +func TestGenerateLexerModeLocalTokenIsNotSharedAcrossModes(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string Content string } + Foo: Greeting=ID "(" Content=INNER ")"; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID + "(" -> push(Inner) + hidden WS + } + token mode Inner { + token INNER: /[A-Z]+/ + ")" -> pop + } + token mode Other { + INNER -> pop + } + `) + assert.Equal(t, 1, strings.Count(code, "var Token_INNER =")) + other := code[strings.Index(code, `NewTokenMode("Other"`):] + assert.NotContains(t, other, "Token_INNER") +} + +func TestGenerateLexerModeLocalTokenGroupWithCommand(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID Closers; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID + "(" -> push(Inner) + hidden WS + } + token mode Inner { + token group Closers { + ")" + "]" + } -> pop + } + `) + assert.Contains(t, code, "var TokenGroup_Closers =") + inner := code[strings.Index(code, `NewTokenMode("Inner"`):] + assert.Contains(t, inner, "lexer.UseTokenType(Keyword_RightParen).WithPopMode(),") + assert.Contains(t, inner, "lexer.UseTokenType(Keyword_RightBracket).WithPopMode(),") +} + +// Regression test for the keyword selector dropping its bookkeeping between +// members: a keyword that is both listed explicitly and matched by a selector +// must be registered once. +func TestGenerateLexerKeywordSelectorDoesNotDuplicateKeywords(t *testing.T) { + code := generateLexerFor(t, ` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting="hello" "world"; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + "hello" + keywords /^[a-z]+$/ + ID + hidden WS + } + `) + assert.Equal(t, 1, strings.Count(code, "lexer.UseTokenType(Keyword_hello)"), + "keyword registered more than once:\n%s", code) + assert.Equal(t, 1, strings.Count(code, "lexer.UseTokenType(Keyword_world)")) +} diff --git a/internal/generator/lexer_util.go b/internal/generator/lexer_util.go new file mode 100644 index 00000000..fe316700 --- /dev/null +++ b/internal/generator/lexer_util.go @@ -0,0 +1,668 @@ +// Copyright 2025 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package generator + +import ( + "context" + "maps" + "regexp" + "slices" + "sort" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/internal/grammar" + "typefox.dev/fastbelt/util/codegen" +) + +// A helper type to manage generator-related information about token types. +// A token type is either originated from +// - a keyword (standalone keyword or token type with keyword content) +// - a token declaration (token type with regexp content) +// - a token group (token type that groups other token types) +type TokenType struct { + // Token index is the unique ID at runtime, used as a constant in + // the generated lexer and parser code. It is also used as the index + // into the token type lookup table. + TokenIndex int + // Name of the token type, used to reflect usage of grammar rules. + // No extra prefix! + Name string + // VarName is the name of the generated variable containing the token type. + VarName string + // Generated code to declare the token type variable. + Code codegen.Node +} + +// Describes how a [TokenType] is used in a [TokenMode]. +// Mind the same naming in the modes package: [modes.TokenTypeUsage]. The concept is the same. Implementation is slightly different. +type TokenTypeUsage struct { + // comment or hidden, look at it like public, protected, private in object-oriented programming. + TokenModifier string + // Command to manipulate the mode stack when this token type is matched. Can be nil. + // Distinguishes between: + // - push(mmode): adds a new mode onto the stack + // - set(mode): replace the current mode with a new one + // - pop: pops the current mode from the stack + Command grammar.TokenCommand +} + +// Belongs to a [TokenMode] and describes how a [TokenType] is used in that mode. +// Actually it would be sufficient to store one array of token type indices, but +// we split this set by category: keywords and regexp tokens. Keywords are matched first, +// so they have priority over regexp tokens. +type TokenTypeIndices struct { + // token indices of keywords that are part of this mode + Keywords []int + // token indices of regexp tokens that are part of this mode + Tokens []int +} + +// A token mode ist an ordered set of token types that are active in a given mode. +type TokenMode struct { + // Unique ID of the token mode, used as a constant. + Id int + // Variable name of the generated token mode constant. + VarName string + // Holds the token indices of the token types that are part of this mode. + TokenTypeIndices TokenTypeIndices + // Holds the token type usage information (modifier and command) for each + // token type in this mode. The key is the token index. + TokenTypeUsages map[int]TokenTypeUsage +} + +// TokenIndexSource is an enum and describes for one token index, from +// which source it was generated. +type TokenIndexSource int + +const ( + // was created by a token declaration (regexp or keyword) + SourceTokenDecl TokenIndexSource = iota + // was created by a keyword (standalone and no token declaration with keyword content) + SourceKeyword + // was created by a token group (so it has members!) + SourceGroup +) + +// TokenIndexLookup is a helper type to manage the mapping between token +// indices and their sources (keyword, token, token group, keyword). +type TokenIndexLookup struct { + // Keyword value to token index mapping. + ByKeyword map[string]int + // Token declaration to token index mapping. + ByToken map[grammar.TokenDecl]int + // Token group to token index mapping. + ByTokenGroup map[grammar.TokenGroup]int + // Token group indices of the members of a token group. The key is the parent token group. + ByTokenGroupParent map[grammar.TokenGroup][]int + // Token index to source type mapping. So that you can find out from which source a token index was generated (keyword, token, token group). + SourceType map[int]TokenIndexSource +} + +// TokenTypeLookup is a helper type to manage the mapping between token types and their token indices. +type TokenTypeLookup struct { + // Get all token types that were created (it can happen that multiple + // token types share the same token index, for example when a keyword + // was declared standalone and in a token declaration). + All []TokenType + // Token index to token type mapping. So that you can find out which token type was generated for a given token index. + ByTokenIndex map[int]*TokenType +} + +// Result of GenerateTokenTypes. It contains all the information about the generated token types, token modes, and their usage in the grammar. +type GenerateTokenTypesResult struct { + // All keywords that were found in the grammar. This includes standalone keywords and keywords that are part of token declarations. + Keywords GetAllKeywordsResult + // All token declarations that were found in the grammar. This includes top-level and mode-level token declarations. + TokenDecls GetAllTokenDeclsResult + // All token groups that were found in the grammar. This includes top-level and mode-level token groups. + TokenGroups GetAllTokenGroupsResult + // All imports that are required for the generated code. Each entry is a package path. + Imports map[string]bool + // Lookup tables for token indices: What token index has X? Where X is a keyword, a token declaration, or a token group. + TokenIndex TokenIndexLookup + // Lookup tables for token types: What token type has X? Where X is a token index. + TokenTypes TokenTypeLookup + // Lookup tables for token modes: What token mode has X? Where X is a mode name (always contains "default"). + TokenModes map[string]*TokenMode + // The order of token modes inside the grammar. Used mostly to keep the order of the generated code stable. + TokenModeOrder []string +} + +func (r GenerateTokenTypesResult) TokenTypeIds() map[string]int { + tokenTypeIds := make(map[string]int, len(r.TokenTypes.All)) + for _, tokenType := range r.TokenTypes.All { + // TokenIndex is the runtime token id emitted by the lexer (the _Idx + // constant). Stores the token index for each token type name + // (which is something like the token rule name). Especially for + // keywords which can be standalone or part of a token declaration, + // this is important to know which token index is used at runtime + // (so: two entries can have the same token index). + // Example why this is important: + // + // grammar XY + // entry Start: "a" A; + // token A: "a" + // + // There are two token types: one for the keyword "a" and one + // for the token declaration A. But since the content is the same, + // they share the same token index. So the generated code will + // have two token types (Keyword_a and Token_A) with the same token + // index, but only one of them will be used as default. + tokenTypeIds[tokenType.Name] = tokenType.TokenIndex + } + return tokenTypeIds +} + +// TokenTypeVarNamesByTokenIndex returns a slice mapping runtime token id +// (TokenIndex) to a token var name. Keyword-backed tokens alias their keyword +// and share its id; later entries (token declarations, groups) override the +// earlier keyword entry so the referenced token name wins. +func (r GenerateTokenTypesResult) TokenTypeVarNamesByTokenIndex() []string { + maxId := 0 + for _, tokenType := range r.TokenTypes.All { + if tokenType.TokenIndex > maxId { + maxId = tokenType.TokenIndex + } + } + names := make([]string, maxId+1) + for _, tokenType := range r.TokenTypes.All { + names[tokenType.TokenIndex] = tokenType.VarName + } + return names +} + +// TokenTypeNamesByTokenIndex returns a slice mapping runtime token id +// (TokenIndex) to a token name. Keyword-backed tokens alias their keyword +// and share its id; later entries (token declarations, groups) override the +// earlier keyword entry so the referenced token name wins. +func (r GenerateTokenTypesResult) TokenTypeNamesByTokenIndex() []string { + maxId := 0 + for _, tokenType := range r.TokenTypes.All { + if tokenType.TokenIndex > maxId { + maxId = tokenType.TokenIndex + } + } + names := make([]string, maxId+1) + for _, tokenType := range r.TokenTypes.All { + names[tokenType.TokenIndex] = tokenType.Name + } + return names +} + +func GenerateTokenTypes(grammr grammar.Grammar) GenerateTokenTypesResult { + keywords := GetAllKeywords(grammr) + tokenDecls := GetAllTokenDecls(grammr) + tokenGroups := GetAllTokenGroups(grammr) + result := GenerateTokenTypesResult{ + Keywords: keywords, + TokenDecls: tokenDecls, + TokenGroups: tokenGroups, + Imports: map[string]bool{}, + TokenIndex: TokenIndexLookup{ + ByKeyword: make(map[string]int), + ByToken: make(map[grammar.TokenDecl]int), + ByTokenGroup: make(map[grammar.TokenGroup]int), + ByTokenGroupParent: make(map[grammar.TokenGroup][]int), + SourceType: make(map[int]TokenIndexSource), + }, + TokenTypes: TokenTypeLookup{ + All: make([]TokenType, 0), + ByTokenIndex: make(map[int]*TokenType), + }, + TokenModes: make(map[string]*TokenMode), + TokenModeOrder: []string{}, + } + populateTokenTypes(&result) + populateTokenModes(&result, grammr.TokenModes()) + return result +} + +func populateTokenModes(result *GenerateTokenTypesResult, tokenModes []grammar.TokenMode) { + keywords := result.Keywords + tokenGroups := result.TokenGroups + tokens := result.TokenDecls + for index, tokenMode := range tokenModes { + modeName := "default" + if !tokenMode.IsDefault() { + modeName = tokenMode.Name() + } + current := TokenMode{ + Id: index, + VarName: "TokenMode_" + modeName, + TokenTypeIndices: TokenTypeIndices{ + Keywords: make([]int, 0), + Tokens: make([]int, 0), + }, + TokenTypeUsages: make(map[int]TokenTypeUsage), + } + result.TokenModes[modeName] = ¤t + // Tracked across all members of the mode: a keyword selector must not + // register a keyword that an earlier member already added. + alreadyAdded := map[int]bool{} + for _, member := range tokenMode.Members() { + pushTokenTypeUsage := func(tokenIndex int, tokenModifier string, command grammar.TokenCommand) { + if _, ok := alreadyAdded[tokenIndex]; !ok { + if source, ok := result.TokenIndex.SourceType[tokenIndex]; ok && source != SourceGroup { + if source == SourceKeyword { + current.TokenTypeIndices.Keywords = append(current.TokenTypeIndices.Keywords, tokenIndex) + } else { + current.TokenTypeIndices.Tokens = append(current.TokenTypeIndices.Tokens, tokenIndex) + } + alreadyAdded[tokenIndex] = true + } else { + return + } + } + if tokenModifier != "" || command != nil { + current.TokenTypeUsages[tokenIndex] = TokenTypeUsage{ + TokenModifier: tokenModifier, + Command: command, + } + } + } + + switch member := member.(type) { + case grammar.TokenDeclUsage: + token := member.Declaration() + tokenIndex := result.TokenIndex.ByToken[token] + pushTokenTypeUsage(tokenIndex, token.Modifier(), token.Command()) + case grammar.TokenGroupUsage: + tokenGroup := member.Group() + for _, tokenIndex := range result.TokenIndex.ByTokenGroupParent[tokenGroup] { + pushTokenTypeUsage(tokenIndex, tokenGroup.Modifier(), tokenGroup.Command()) + } + case grammar.KeywordUsage: + tokenIndex := result.TokenIndex.ByKeyword[member.Keyword().Value()] + pushTokenTypeUsage(tokenIndex, member.Modifier(), member.Command()) + case grammar.TokenUsage: + token := member.TokenRef().Ref(context.Background()) + + overrideUsage := func(tokenIndex int) { + if member.Modifier() != "" || member.Command() != nil { + current.TokenTypeUsages[tokenIndex] = TokenTypeUsage{ + TokenModifier: member.Modifier(), + Command: member.Command(), + } + } + } + + var tokenIndex int + switch rule := token.(type) { + case grammar.TokenDecl: + tokenIndex = result.TokenIndex.ByToken[rule] + pushTokenTypeUsage(tokenIndex, rule.Modifier(), rule.Command()) + overrideUsage(tokenIndex) + case grammar.TokenGroup: + for _, tokenIndex := range result.TokenIndex.ByTokenGroupParent[rule] { + pushTokenTypeUsage(tokenIndex, rule.Modifier(), rule.Command()) + overrideUsage(tokenIndex) + } + } + case grammar.KeywordSelector: + pattern := regexp.MustCompile(grammar.RegexpValue(member.Selector())) + for _, keyword := range keywords.Keywords { + tokenIndex := result.TokenIndex.ByKeyword[keyword.Value()] + value := grammar.KeywordValue(keyword) + if !alreadyAdded[tokenIndex] && pattern.MatchString(value) { + current.TokenTypeIndices.Keywords = append(current.TokenTypeIndices.Keywords, tokenIndex) + alreadyAdded[tokenIndex] = true + } + } + } + } + result.TokenModeOrder = append(result.TokenModeOrder, modeName) + } + if result.TokenModes["default"] == nil { + //if token mode "default" is not defined, we need to create it + defaultMode := TokenMode{ + Id: len(result.TokenModes), + VarName: "TokenMode_default", + TokenTypeIndices: TokenTypeIndices{ + Keywords: make([]int, 0), + Tokens: make([]int, 0), + }, + TokenTypeUsages: make(map[int]TokenTypeUsage), + } + result.TokenModes["default"] = &defaultMode + result.TokenModeOrder = append(result.TokenModeOrder, "default") + + for _, tokenGroup := range tokenGroups.TopLevel { + for _, tokenIndex := range result.TokenIndex.ByTokenGroupParent[tokenGroup] { + if source, ok := result.TokenIndex.SourceType[tokenIndex]; ok && source != SourceGroup { + if source == SourceKeyword { + if slices.Contains(defaultMode.TokenTypeIndices.Keywords, tokenIndex) { + continue + } + defaultMode.TokenTypeIndices.Keywords = append(defaultMode.TokenTypeIndices.Keywords, tokenIndex) + } else { + if slices.Contains(defaultMode.TokenTypeIndices.Tokens, tokenIndex) { + continue + } + defaultMode.TokenTypeIndices.Tokens = append(defaultMode.TokenTypeIndices.Tokens, tokenIndex) + } + if tokenGroup.Modifier() != "" || tokenGroup.Command() != nil { + defaultMode.TokenTypeUsages[tokenIndex] = TokenTypeUsage{ + TokenModifier: tokenGroup.Modifier(), + Command: tokenGroup.Command(), + } + } + } + } + } + + for _, keyword := range keywords.Keywords { + tokenIndex := result.TokenIndex.ByKeyword[keyword.Value()] + if slices.Contains(defaultMode.TokenTypeIndices.Keywords, tokenIndex) { + continue + } + defaultMode.TokenTypeIndices.Keywords = append(defaultMode.TokenTypeIndices.Keywords, tokenIndex) + //keywords don't have type or command, so we don't need to add anything to TokenTypeUsages + } + + for _, token := range tokens.TopLevel { + tokenIndex := result.TokenIndex.ByToken[token] + if result.TokenIndex.SourceType[tokenIndex] != SourceTokenDecl { + continue + } + if !slices.Contains(defaultMode.TokenTypeIndices.Tokens, tokenIndex) { + defaultMode.TokenTypeIndices.Tokens = append(defaultMode.TokenTypeIndices.Tokens, tokenIndex) + } + if token.Modifier() != "" || token.Command() != nil { + defaultMode.TokenTypeUsages[tokenIndex] = TokenTypeUsage{ + TokenModifier: token.Modifier(), + Command: token.Command(), + } + } + } + } +} + +func populateTokenTypes(result *GenerateTokenTypesResult) { + keywords := result.Keywords + tokens := result.TokenDecls + tokenGroups := result.TokenGroups + + // Starting with 1 - prevent clash with EOF (index 0) + tokenIndex := 1 + for _, keyword := range keywords.Keywords { + code := generateKeywordTokenType(keyword, tokenIndex) + mergeImports(result.Imports, code.Imports) + tokenType := TokenType{ + TokenIndex: tokenIndex, + VarName: GeneratedTokenName(keyword), + Name: keyword.Value(), + Code: code.Code, + } + result.TokenTypes.All = append(result.TokenTypes.All, tokenType) + result.TokenTypes.ByTokenIndex[tokenIndex] = &tokenType + result.TokenIndex.ByKeyword[keyword.Value()] = tokenIndex + result.TokenIndex.SourceType[tokenIndex] = SourceKeyword + tokenIndex++ + } + for _, token := range tokens.All { + varName := GeneratedTokenName(token) + var currentTokenIndex int + switch element := token.Content().(type) { + case grammar.KeywordTokenContent: + keywordIndex := keywords.ByValue[element.Keyword().Value()] + keyword := keywords.Keywords[keywordIndex] + code := codegen.NewNode() + code.AppendLine("const ", GeneratedTokenIdxName(token), " = ", GeneratedTokenIdxName(keyword)) + code.AppendLine() + code.AppendLine("var ", varName, " = ", GeneratedTokenName(keyword)) + mergeImports(result.Imports, map[string]bool{}) + currentTokenIndex = result.TokenIndex.ByKeyword[keyword.Value()] + result.TokenIndex.SourceType[currentTokenIndex] = SourceTokenDecl + tokenType := TokenType{ + TokenIndex: currentTokenIndex, + VarName: varName, + Name: token.Name(), + Code: code, + } + result.TokenTypes.All = append(result.TokenTypes.All, tokenType) + result.TokenTypes.ByTokenIndex[currentTokenIndex] = &tokenType + result.TokenIndex.ByToken[token] = currentTokenIndex + case grammar.RegexpTokenContent: + lexerResult := generateRegexpTokenElement(token, element, tokenIndex) + mergeImports(result.Imports, lexerResult.Imports) + currentTokenIndex = tokenIndex + tokenType := TokenType{ + TokenIndex: currentTokenIndex, + VarName: varName, + Name: token.Name(), + Code: lexerResult.Code, + } + result.TokenTypes.All = append(result.TokenTypes.All, tokenType) + result.TokenTypes.ByTokenIndex[currentTokenIndex] = &tokenType + result.TokenIndex.ByToken[token] = currentTokenIndex + result.TokenIndex.SourceType[currentTokenIndex] = SourceTokenDecl + tokenIndex++ + } + } + tokenGroupMembers := map[string][]string{} + for _, tokenGroup := range tokenGroups.All { + tokenGroupMembers[tokenGroup.Name()] = getAllTokenGroupMembers(tokenGroup, keywords) + } + // Token groups need to be topologically sorted, so that nested groups appear after their members + sortedTokenGroups := sortTokenGroups(tokenGroups.All, tokenGroupMembers) + for _, tokenGroup := range sortedTokenGroups { + lexerResult := generateTokenGroupType(tokenGroup, tokenGroupMembers, tokenIndex) + mergeImports(result.Imports, lexerResult.Imports) + tokenType := TokenType{ + TokenIndex: tokenIndex, + VarName: GeneratedTokenName(tokenGroup), + Name: tokenGroup.Name(), + Code: lexerResult.Code, + } + result.TokenTypes.All = append(result.TokenTypes.All, tokenType) + result.TokenTypes.ByTokenIndex[tokenIndex] = &tokenType + result.TokenIndex.ByTokenGroup[tokenGroup] = tokenIndex + result.TokenIndex.SourceType[tokenIndex] = SourceGroup + tokenIndex++ + } + result.TokenIndex.ByTokenGroupParent = getAllTokenGroupMemberTokenIndices(sortedTokenGroups, keywords, result.TokenIndex) +} + +func mergeImports(target map[string]bool, source map[string]bool) { + maps.Copy(target, source) +} + +func sortTokenGroups(tokenGroups []grammar.TokenGroup, members map[string][]string) []grammar.TokenGroup { + names := make([]string, len(tokenGroups)) + for i, tg := range tokenGroups { + names[i] = tg.Name() + } + sort.Strings(names) + topoSorted := topoSort(names, members, false) + sortedGroups := make([]grammar.TokenGroup, len(tokenGroups)) + for i, name := range topoSorted { + for _, tg := range tokenGroups { + if tg.Name() == name { + sortedGroups[i] = tg + break + } + } + } + return sortedGroups +} + +type GetAllKeywordsResult struct { + Keywords []grammar.Keyword + ByValue map[string]int +} + +func GetAllKeywords(grammr grammar.Grammar) GetAllKeywordsResult { + keywords := map[string]grammar.Keyword{} + allNodes := []grammar.Keyword{} + for node := range core.AllChildren(grammr) { + if keyword, ok := node.(grammar.Keyword); ok { + keywords[keyword.Value()] = keyword + allNodes = append(allNodes, keyword) + } + } + return keysFromMap(keywords, allNodes) +} + +type GetAllTokenDeclsResult struct { + TopLevel []grammar.TokenDecl + ModeLevel []grammar.TokenDecl + All []grammar.TokenDecl +} + +func GetAllTokenDecls(grammr grammar.Grammar) GetAllTokenDeclsResult { + topLevel := []grammar.TokenDecl{} + modeLevel := []grammar.TokenDecl{} + for node := range core.AllChildren(grammr) { + if tokenDecl, ok := node.(grammar.TokenDecl); ok { + _, ok := tokenDecl.Container().(grammar.TokenDeclUsage) + if ok { + modeLevel = append(modeLevel, tokenDecl) + } else { + topLevel = append(topLevel, tokenDecl) + } + } + } + length := len(topLevel) + len(modeLevel) + all := make([]grammar.TokenDecl, length) + copy(all, topLevel) + copy(all[len(topLevel):], modeLevel) + return GetAllTokenDeclsResult{ + All: all, + TopLevel: topLevel, + ModeLevel: modeLevel, + } +} + +type GetAllTokenGroupsResult struct { + All []grammar.TokenGroup + TopLevel []grammar.TokenGroup + ModeLevel []grammar.TokenGroup +} + +func GetAllTokenGroups(grammr grammar.Grammar) GetAllTokenGroupsResult { + topLevel := []grammar.TokenGroup{} + modeLevel := []grammar.TokenGroup{} + + for node := range core.AllChildren(grammr) { + if tokenGroup, ok := node.(grammar.TokenGroup); ok { + _, ok := tokenGroup.Container().(grammar.TokenGroupUsage) + if ok { + modeLevel = append(modeLevel, tokenGroup) + } else { + topLevel = append(topLevel, tokenGroup) + } + } + } + + length := len(topLevel) + len(modeLevel) + all := make([]grammar.TokenGroup, length) + copy(all, topLevel) + copy(all[len(topLevel):], modeLevel) + return GetAllTokenGroupsResult{ + All: all, + TopLevel: topLevel, + ModeLevel: modeLevel, + } +} + +func keysFromMap(m map[string]grammar.Keyword, allNodes []grammar.Keyword) GetAllKeywordsResult { + keywords := []grammar.Keyword{} + for _, v := range m { + keywords = append(keywords, v) + } + sort.Slice(keywords, func(i, j int) bool { + return keywords[i].Value() < keywords[j].Value() + }) + byValue := map[string]int{} + for i, k := range keywords { + byValue[k.Value()] = i + } + return GetAllKeywordsResult{ + Keywords: keywords, + ByValue: byValue, + } +} + +func GeneratedTokenName(t core.AstNode) string { + switch t := t.(type) { + case grammar.TokenDecl: + return "Token_" + t.Name() + case grammar.TokenGroup: + return "TokenGroup_" + t.Name() + case grammar.Keyword: + return "Keyword_" + grammar.KeywordName(t) + default: + panic("unexpected type") + } +} + +func GeneratedTokenIdxName(t core.AstNode) string { + return GeneratedTokenName(t) + "_Idx" +} + +func getAllTokenGroupMemberTokenIndices(sortedTokenGroups []grammar.TokenGroup, keywords GetAllKeywordsResult, lookup TokenIndexLookup) map[grammar.TokenGroup][]int { + tokenGroupMembers := map[grammar.TokenGroup][]int{} + for _, tokenGroup := range sortedTokenGroups { + tokenGroupMembers[tokenGroup] = []int{} + } + for _, tokenGroup := range sortedTokenGroups { + for _, tokenRef := range tokenGroup.TokenRefs() { + tokenRule := tokenRef.Ref(context.Background()) + if tokenRule != nil { + if subTokenGroup, ok := tokenRule.(grammar.TokenGroup); ok { + tokenGroupMembers[tokenGroup] = append(tokenGroupMembers[tokenGroup], tokenGroupMembers[subTokenGroup]...) + } else if tokenDecl, ok := tokenRule.(grammar.TokenDecl); ok { + if idx, ok := lookup.ByToken[tokenDecl]; ok { + tokenGroupMembers[tokenGroup] = append(tokenGroupMembers[tokenGroup], idx) + } + } + } + } + for _, selector := range tokenGroup.KeywordSelectors() { + pattern := regexp.MustCompile(grammar.RegexpValue(selector.Image)) + for _, keyword := range keywords.Keywords { + if idx, ok := lookup.ByKeyword[keyword.Value()]; ok { + if pattern.MatchString(keyword.Value()) { + tokenGroupMembers[tokenGroup] = append(tokenGroupMembers[tokenGroup], idx) + } + } + } + } + for _, keyword := range tokenGroup.Keywords() { + if idx, ok := lookup.ByKeyword[keyword.Value()]; ok { + tokenGroupMembers[tokenGroup] = append(tokenGroupMembers[tokenGroup], idx) + } + } + } + return tokenGroupMembers +} + +func getAllTokenGroupMembers(tokenGroup grammar.TokenGroup, keywords GetAllKeywordsResult) []string { + members := map[string]bool{} + for _, tokenRef := range tokenGroup.TokenRefs() { + tokenRule := tokenRef.Ref(context.Background()) + if tokenRule != nil { + name := GeneratedTokenName(tokenRule) + members[name] = true + } + } + for _, selector := range tokenGroup.KeywordSelectors() { + pattern := regexp.MustCompile(grammar.RegexpValue(selector.Image)) + for _, keyword := range keywords.Keywords { + name := GeneratedTokenName(keyword) + value := grammar.KeywordValue(keyword) + if !members[name] && pattern.MatchString(value) { + members[name] = true + } + } + } + for _, keyword := range tokenGroup.Keywords() { + name := GeneratedTokenName(keyword) + members[name] = true + } + slice := slices.Collect(maps.Keys(members)) + sort.Strings(slice) + return slice +} diff --git a/internal/generator/lexer_util_test.go b/internal/generator/lexer_util_test.go new file mode 100644 index 00000000..dfa620fa --- /dev/null +++ b/internal/generator/lexer_util_test.go @@ -0,0 +1,268 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package generator + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "typefox.dev/fastbelt/internal/grammar" + "typefox.dev/fastbelt/test" +) + +func TestGetAllKeywords_InRule(t *testing.T) { + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(` + grammar Test; + interface Model { + Greeting string + } + Model: Greeting="hello" "world" | Greeting="hi" "there" + `).AssertNoErrors() + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + result := GetAllKeywords(grammr) + require.Len(t, result.Keywords, 4) + assert.Contains(t, result.ByValue, "\"hello\"") + assert.Contains(t, result.ByValue, "\"hi\"") + assert.Contains(t, result.ByValue, "\"there\"") + assert.Contains(t, result.ByValue, "\"world\"") +} + +func TestGetAllKeywords_InTokenDeclaration(t *testing.T) { + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(` + grammar Test; + interface Model { + Greeting string + } + Model: Greeting=HELLO "world" + token HELLO: "hello" + `).AssertNoErrors() + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + result := GetAllKeywords(grammr) + require.Len(t, result.Keywords, 2) + assert.Contains(t, result.ByValue, "\"hello\"") + assert.Contains(t, result.ByValue, "\"world\"") +} + +func TestGetAllKeywords_InTokenGroup(t *testing.T) { + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(` + grammar Test; + interface Model { + Greeting string + } + Model: Greeting=GREETING "world" + token group GREETING { + "hello" + "hi" + } + `).AssertNoErrors() + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + result := GetAllKeywords(grammr) + require.Len(t, result.Keywords, 3) + assert.Contains(t, result.ByValue, "\"hello\"") + assert.Contains(t, result.ByValue, "\"hi\"") + assert.Contains(t, result.ByValue, "\"world\"") +} + +func TestGetAllTokenDecls_SplitsTopLevelAndModeLevel(t *testing.T) { + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(` + grammar Test; + interface Model { + Greeting string + } + Model: Greeting=ID INNER + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID + hidden WS + } + token mode Inner { + token INNER: /[A-Z]+/ + } + `) + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + result := GetAllTokenDecls(grammr) + assert.Equal(t, []string{"ID", "WS"}, tokenDeclNames(result.TopLevel)) + assert.Equal(t, []string{"INNER"}, tokenDeclNames(result.ModeLevel)) + // All lists top-level declarations first, then the mode-local ones. + assert.Equal(t, []string{"ID", "WS", "INNER"}, tokenDeclNames(result.All)) +} + +func tokenDeclNames(decls []grammar.TokenDecl) []string { + names := make([]string, len(decls)) + for i, decl := range decls { + names[i] = decl.Name() + } + return names +} + +func TestGetAllTokenGroups_SplitsTopLevelAndModeLevel(t *testing.T) { + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(` + grammar Test; + interface Model { + Greeting string + } + Model: Greeting=Outer Closers + token group Outer { ID } + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + Outer + hidden WS + } + token mode Inner { + token group Closers { ")" } + } + `) + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + result := GetAllTokenGroups(grammr) + assert.Equal(t, []string{"Outer"}, tokenGroupNames(result.TopLevel)) + assert.Equal(t, []string{"Closers"}, tokenGroupNames(result.ModeLevel)) + assert.Equal(t, []string{"Outer", "Closers"}, tokenGroupNames(result.All)) +} + +func tokenGroupNames(groups []grammar.TokenGroup) []string { + names := make([]string, len(groups)) + for i, group := range groups { + names[i] = group.Name() + } + return names +} + +func TestGetAllKeywords_InTokenMode(t *testing.T) { + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(` + grammar Test; + interface Model { + Greeting string + } + Model: Greeting="hello" "world" + token mode default { + "hello" + "world" + } + `).AssertNoErrors() + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + result := GetAllKeywords(grammr) + require.Len(t, result.Keywords, 2) + assert.Contains(t, result.ByValue, "\"hello\"") + assert.Contains(t, result.ByValue, "\"world\"") +} + +func TestPopulateTokenModes_ShouldAddModifiersOnTokenElements(t *testing.T) { + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(` + grammar Test; + interface Model { + Greeting string + } + Model: Greeting=Hello + token Hello: "hello" + hidden token World: "world" + hidden token User: /user/ + `).AssertNoErrors() + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + tokenTypes := GenerateTokenTypes(grammr) + populateTokenTypes(&tokenTypes) + populateTokenModes(&tokenTypes, grammr.TokenModes()) + tokenDecls := grammr.Terminals() + defaultMode := tokenTypes.TokenModes["default"] + indexHello := tokenTypes.TokenIndex.ByToken[tokenDecls[0]] + indexWorld := tokenTypes.TokenIndex.ByToken[tokenDecls[1]] + indexUser := tokenTypes.TokenIndex.ByToken[tokenDecls[2]] + assert.Equal(t, 2, len(defaultMode.TokenTypeIndices.Keywords)) + assert.Equal(t, "hidden", defaultMode.TokenTypeUsages[indexWorld].TokenModifier) + assert.Equal(t, "hidden", defaultMode.TokenTypeUsages[indexUser].TokenModifier) + _, ok = defaultMode.TokenTypeUsages[indexHello] + assert.False(t, ok) +} + +func TestPopulateTokenModes_ShouldHandleTopLevelTokenGroupProperlyInTokenMode(t *testing.T) { + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(` + grammar Test; + + interface Model { + Greeting string + } + entry Model: "x" Greeting=X + + token group X { + "x" + } + + token mode default { + "x" -> push(Y) + } + + token mode Y { + X -> pop + } + `).AssertNoErrors() + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + tokenTypes := GenerateTokenTypes(grammr) + populateTokenTypes(&tokenTypes) + populateTokenModes(&tokenTypes, grammr.TokenModes()) + + topLevelTokenGroups := grammr.TokenGroups() + require.Len(t, topLevelTokenGroups, 1) + indexKeyword := tokenTypes.TokenIndex.ByKeyword["\"x\""] + + defaultMode := tokenTypes.TokenModes["default"] + assert.Equal(t, 1, len(defaultMode.TokenTypeIndices.Keywords)) + assert.Equal(t, indexKeyword, defaultMode.TokenTypeIndices.Keywords[0]) + assert.Equal(t, 1, len(defaultMode.TokenTypeUsages)) + keywordUsage, ok := defaultMode.TokenTypeUsages[indexKeyword] + assert.True(t, ok) + assert.Equal(t, "push", keywordUsage.Command.Type()) + assert.Equal(t, "Y", keywordUsage.Command.Mode().Ref(context.Background()).Name()) + + yMode := tokenTypes.TokenModes["Y"] + assert.Equal(t, 1, len(yMode.TokenTypeIndices.Keywords)) + assert.Equal(t, indexKeyword, yMode.TokenTypeIndices.Keywords[0]) + assert.Equal(t, 1, len(yMode.TokenTypeUsages)) + keywordUsage2, ok := yMode.TokenTypeUsages[indexKeyword] + assert.True(t, ok) + assert.Equal(t, "pop", keywordUsage2.Command.Type()) + assert.Nil(t, keywordUsage2.Command.Mode()) +} + +func TestPopulateTokenTypes_ShouldWorkThanksToTopologicalSort(t *testing.T) { + f := test.New(t, grammar.CreateServices()) + doc := f.Parse(` + grammar Test; + interface Model { + Greeting string + } + Model: Greeting=Outer + token group Outer { Inner } + token group Inner { "word" } + `).AssertNoErrors() + grammr, ok := doc.Document.Root.(grammar.Grammar) + require.True(t, ok) + tokenTypes := GenerateTokenTypes(grammr) + populateTokenTypes(&tokenTypes) + topLevelTokenGroups := grammr.TokenGroups() + outerGroup := topLevelTokenGroups[0] + innerGroup := topLevelTokenGroups[1] + + assert.EqualValues(t, []int{1}, tokenTypes.TokenIndex.ByTokenGroupParent[outerGroup]) + assert.EqualValues(t, []int{1}, tokenTypes.TokenIndex.ByTokenGroupParent[innerGroup]) +} diff --git a/internal/generator/parser_generator.go b/internal/generator/parser_generator.go index 7472a95d..f5384492 100644 --- a/internal/generator/parser_generator.go +++ b/internal/generator/parser_generator.go @@ -29,7 +29,7 @@ type parserATNData struct { loopAdaptive map[core.AstNode]int // grammar.Element -> ALL(*) decision index orDecision map[grammar.Element]*internalATN.ATNState // grammar.Alternatives -> alternative-choice state loopDecision map[grammar.Element]*internalATN.ATNState // grammar.Element -> cardinality-guard state - tokenVarNames []string // ATN token id (0-based) -> generated token var name + tokenVarNames []string // ATN token id (TokenIndex) -> generated token var name } // BuildParserATNData builds the ATN and all derived name/decision maps used by @@ -37,7 +37,7 @@ type parserATNData struct { // GenerateParser, GenerateCompletionParser, and GenerateParserLookahead. // Returns nil when the ATN cannot be built (invalid grammar). func BuildParserATNData(grammr grammar.Grammar, tokenTypes GenerateTokenTypesResult) *parserATNData { - builtATN, _ := internalATN.CreateATN(grammr, tokenTypes.TokenTypeIds) + builtATN, _ := internalATN.CreateATN(grammr, tokenTypes.TokenTypeIds()) if builtATN == nil { return nil } @@ -54,7 +54,7 @@ func BuildParserATNData(grammr grammar.Grammar, tokenTypes GenerateTokenTypesRes loopAdaptive: loopAdaptive, orDecision: builtATN.OrDecision, loopDecision: builtATN.LoopDecision, - tokenVarNames: tokenTypes.TokenTypeVarNames, + tokenVarNames: tokenTypes.TokenTypeVarNamesByTokenIndex(), } } @@ -317,7 +317,7 @@ func GenerateParserLookahead(grammr grammar.Grammar, packageName string, tokenTy // Build the maps once; generateLL1Lookahead needs them per table. varNameToId := buildVarNameToId(tokenTypes) - groupMembers := buildGroupVarNameToMembers(grammr) + groupMembers := buildGroupVarNameToMembers(grammr, tokenTypes.Keywords) node := NewRootNode() node.AppendLine("package ", packageName) @@ -440,6 +440,13 @@ func generateLookaheadMethods(context *ParserGeneratorContext) []lookaheadMethod for _, tg := range context.grammar.TokenGroups() { groupNames[GeneratedTokenName(tg)] = true } + for _, tm := range context.grammar.TokenModes() { + for _, member := range tm.Members() { + if tgu, ok := member.(grammar.TokenGroupUsage); ok { + groupNames[GeneratedTokenName(tgu.Group())] = true + } + } + } for el, lv := range context.lookaheads { // The adaptive decision map is keyed by the assignable value for @@ -1541,20 +1548,30 @@ func (d *parserATNData) firstSetOption(state *internalATN.ATNState) []string { } func buildVarNameToId(tokenTypes GenerateTokenTypesResult) map[string]int { - varNameToId := make(map[string]int, len(tokenTypes.TokenTypeVarNames)) - for i, varName := range tokenTypes.TokenTypeVarNames { - varNameToId[varName] = i + 1 // token IDs start at 1 + varNameToId := make(map[string]int, len(tokenTypes.TokenTypes.All)) + for _, tokenType := range tokenTypes.TokenTypes.All { + // Use the runtime token id (TokenIndex), matching la.TypeId at runtime. + // Keyword-backed tokens alias their keyword and share its id. + varNameToId[tokenType.VarName] = tokenType.TokenIndex } return varNameToId } -func buildGroupVarNameToMembers(grammr grammar.Grammar) map[string][]string { - keywords := GetAllKeywords(grammr) +func buildGroupVarNameToMembers(grammr grammar.Grammar, keywords GetAllKeywordsResult) map[string][]string { groups := make(map[string][]string) for _, tg := range grammr.TokenGroups() { varName := GeneratedTokenName(tg) groups[varName] = getAllTokenGroupMembers(tg, keywords) } + for _, tm := range grammr.TokenModes() { + for _, member := range tm.Members() { + if tgu, ok := member.(grammar.TokenGroupUsage); ok { + tg := tgu.Group() + varName := GeneratedTokenName(tg) + groups[varName] = getAllTokenGroupMembers(tg, keywords) + } + } + } return groups } @@ -1579,7 +1596,14 @@ func getLeafIdsForVarName(varName string, varNameToId map[string]int, groupMembe } func generateLL1Lookahead(node codegen.Node, name string, lookahead LL1Decision, varNameToId map[string]int, groupMembers map[string][]string) { - maxId := len(varNameToId) // IDs are 1..N + // Token ids may be sparse (keyword-backed tokens alias their keyword), so + // size the lookup by the largest id rather than the number of var names. + maxId := 0 + for _, id := range varNameToId { + if id > maxId { + maxId = id + } + } lookup := make([]int, maxId+1) for i := range lookup { lookup[i] = -1 diff --git a/internal/generator/util.go b/internal/generator/util.go index 6381131b..5745134c 100644 --- a/internal/generator/util.go +++ b/internal/generator/util.go @@ -7,11 +7,8 @@ package generator import ( "go/format" "runtime" - "sort" "strings" - core "typefox.dev/fastbelt" - "typefox.dev/fastbelt/internal/grammar" "typefox.dev/fastbelt/util/codegen" ) @@ -56,46 +53,6 @@ func WriteSB(sb *strings.Builder, texts ...string) { } } -func GetAllKeywords(grammr grammar.Grammar) []grammar.Keyword { - keywords := map[string]grammar.Keyword{} - for node := range core.AllChildren(grammr) { - if keyword, ok := node.(grammar.Keyword); ok { - keywords[keyword.Value()] = keyword - } - } - return keysFromMap(keywords) -} - -func keysFromMap(m map[string]grammar.Keyword) []grammar.Keyword { - keys := []grammar.Keyword{} - for _, v := range m { - keys = append(keys, v) - } - sort.Slice(keys, func(i, j int) bool { - return keys[i].Value() < keys[j].Value() - }) - return keys -} - -func GeneratedTokenName(t core.AstNode) string { - switch t := t.(type) { - case grammar.AbstractTokenRule: - return "Token_" + t.Name() - case grammar.Keyword: - return "Keyword_" + grammar.KeywordName(t) - default: - panic("unexpected type") - } -} - -func GeneratedTokenIdxName(t core.AstNode) string { - return GeneratedTokenName(t) + "_Idx" -} - -func KeywordValue(k grammar.Keyword) string { - return k.Value()[1 : len(k.Value())-1] -} - // topoSort returns uniqueTargets sorted so that child elements appear before // their parent element. func topoSort(uniqueTargets []string, parents map[string][]string, ascending bool) []string { diff --git a/internal/grammar/atn.md b/internal/grammar/atn.md index 492aa344..a59ce87e 100644 --- a/internal/grammar/atn.md +++ b/internal/grammar/atn.md @@ -6,54 +6,59 @@ flowchart TD q0(["Grammar__Start (0)
RuleStart"]) q1(["Grammar__Stop (1)
RuleStop"]) - q52["Grammar_grammar (52)
Basic
"] - q53["Grammar_Name_ID (53)
Basic
"] - q54["Grammar_Semicolon (54)
Basic
"] - q55["Grammar__Basic_0 (55)
Basic
"] - q56{"Grammar__Basic_1 (56)
Basic

dec=0"} - q57["Grammar__Basic_2 (57)
Basic
"] - q58["Grammar__Basic_3 (58)
Basic
"] - q59["Grammar__Basic_4 (59)
Basic
"] - q60["Grammar__Basic_5 (60)
Basic
"] - q61["Grammar__Basic_6 (61)
Basic
"] - q62["Grammar__Basic_7 (62)
Basic
"] - q63["Grammar__Basic_8 (63)
Basic
"] - q64["Grammar__Basic_9 (64)
Basic
"] - q65["Grammar__Basic_10 (65)
Basic
"] - q66["Grammar__Basic_11 (66)
Basic
"] - q67{"Grammar__Basic_12 (67)
Basic

dec=1"} - q68["Grammar__BlockEnd (68)
BlockEnd
"] - q69{"Grammar__LoopEntry (69)
LoopEntry

dec=2"} - q70["Grammar__LoopEnd (70)
LoopEnd
"] - q71["Grammar__LoopBack (71)
LoopBack
"] - - q0 --> q52 - q52 -->|"tok("grammar")"| q53 - q53 -->|"tok(ID)"| q56 - q54 -->|"tok(";")"| q55 - q55 --> q69 - q56 --> q54 - q56 --> q55 - q57 -.->|"[ParserRule]"| q58 - q58 --> q68 - q59 -.->|"[Token]"| q60 - q60 --> q68 - q61 -.->|"[TokenGroup]"| q62 - q62 --> q68 - q63 -.->|"[Interface]"| q64 - q64 --> q68 - q65 -.->|"[CompositeRule]"| q66 - q66 --> q68 - q67 --> q57 - q67 --> q59 - q67 --> q61 - q67 --> q63 - q67 --> q65 - q68 --> q71 - q69 --> q67 - q69 --> q70 - q70 --> q1 - q71 --> q69 + q74["Grammar_grammar (74)
Basic
"] + q75["Grammar_Name_ID (75)
Basic
"] + q76["Grammar_Semicolon (76)
Basic
"] + q77["Grammar__Basic_0 (77)
Basic
"] + q78{"Grammar__Basic_1 (78)
Basic

dec=0"} + q79["Grammar__Basic_2 (79)
Basic
"] + q80["Grammar__Basic_3 (80)
Basic
"] + q81["Grammar__Basic_4 (81)
Basic
"] + q82["Grammar__Basic_5 (82)
Basic
"] + q83["Grammar__Basic_6 (83)
Basic
"] + q84["Grammar__Basic_7 (84)
Basic
"] + q85["Grammar__Basic_8 (85)
Basic
"] + q86["Grammar__Basic_9 (86)
Basic
"] + q87["Grammar__Basic_10 (87)
Basic
"] + q88["Grammar__Basic_11 (88)
Basic
"] + q89["Grammar__Basic_12 (89)
Basic
"] + q90["Grammar__Basic_13 (90)
Basic
"] + q91{"Grammar__Basic_14 (91)
Basic

dec=1"} + q92["Grammar__BlockEnd (92)
BlockEnd
"] + q93{"Grammar__LoopEntry (93)
LoopEntry

dec=2"} + q94["Grammar__LoopEnd (94)
LoopEnd
"] + q95["Grammar__LoopBack (95)
LoopBack
"] + + q0 --> q74 + q74 -->|"tok("grammar")"| q75 + q75 -->|"tok(ID)"| q78 + q76 -->|"tok(";")"| q77 + q77 --> q93 + q78 --> q76 + q78 --> q77 + q79 -.->|"[ParserRule]"| q80 + q80 --> q92 + q81 -.->|"[TokenDecl]"| q82 + q82 --> q92 + q83 -.->|"[TokenGroup]"| q84 + q84 --> q92 + q85 -.->|"[TokenMode]"| q86 + q86 --> q92 + q87 -.->|"[Interface]"| q88 + q88 --> q92 + q89 -.->|"[CompositeRule]"| q90 + q90 --> q92 + q91 --> q79 + q91 --> q81 + q91 --> q83 + q91 --> q85 + q91 --> q87 + q91 --> q89 + q92 --> q95 + q93 --> q91 + q93 --> q94 + q94 --> q1 + q95 --> q93 ``` ## Interface @@ -62,49 +67,49 @@ flowchart TD flowchart TD q2(["Interface__Start (2)
RuleStart"]) q3(["Interface__Stop (3)
RuleStop"]) - q72["Interface_interface (72)
Basic
"] - q73["Interface_Name_ID (73)
Basic
"] - q74["Interface_extends (74)
Basic
"] - q75["Interface_Extends_ID_0 (75)
Basic
"] - q76["Interface_Comma (76)
Basic
"] - q77["Interface_Extends_ID_1 (77)
Basic
"] - q78["Interface__Basic_0 (78)
Basic
"] - q79{"Interface__LoopEntry_0 (79)
LoopEntry

dec=3"} - q80["Interface__LoopEnd_0 (80)
LoopEnd
"] - q81["Interface__LoopBack_0 (81)
LoopBack
"] - q82{"Interface__Basic_1 (82)
Basic

dec=4"} - q83["Interface_LeftBrace (83)
Basic
"] - q84["Interface__Basic_2 (84)
Basic
"] - q85["Interface__Basic_3 (85)
Basic
"] - q86{"Interface__LoopEntry_1 (86)
LoopEntry

dec=5"} - q87["Interface__LoopEnd_1 (87)
LoopEnd
"] - q88["Interface__LoopBack_1 (88)
LoopBack
"] - q89["Interface_RightBrace (89)
Basic
"] - q90["Interface__Basic_4 (90)
Basic
"] - - q2 --> q72 - q72 -->|"tok("interface")"| q73 - q73 -->|"tok(ID)"| q82 - q74 -->|"tok("extends")"| q75 - q75 -->|"tok(ID)"| q79 - q76 -->|"tok(",")"| q77 - q77 -->|"tok(ID)"| q78 - q78 --> q81 - q79 --> q76 - q79 --> q80 - q80 --> q83 - q81 --> q79 - q82 --> q74 - q82 --> q80 - q83 -->|"tok("{")"| q86 - q84 -.->|"[Field]"| q85 - q85 --> q88 - q86 --> q84 - q86 --> q87 - q87 --> q89 - q88 --> q86 - q89 -->|"tok("}")"| q90 - q90 --> q3 + q96["Interface_interface (96)
Basic
"] + q97["Interface_Name_ID (97)
Basic
"] + q98["Interface_extends (98)
Basic
"] + q99["Interface_Extends_ID_0 (99)
Basic
"] + q100["Interface_Comma (100)
Basic
"] + q101["Interface_Extends_ID_1 (101)
Basic
"] + q102["Interface__Basic_0 (102)
Basic
"] + q103{"Interface__LoopEntry_0 (103)
LoopEntry

dec=3"} + q104["Interface__LoopEnd_0 (104)
LoopEnd
"] + q105["Interface__LoopBack_0 (105)
LoopBack
"] + q106{"Interface__Basic_1 (106)
Basic

dec=4"} + q107["Interface_LeftBrace (107)
Basic
"] + q108["Interface__Basic_2 (108)
Basic
"] + q109["Interface__Basic_3 (109)
Basic
"] + q110{"Interface__LoopEntry_1 (110)
LoopEntry

dec=5"} + q111["Interface__LoopEnd_1 (111)
LoopEnd
"] + q112["Interface__LoopBack_1 (112)
LoopBack
"] + q113["Interface_RightBrace (113)
Basic
"] + q114["Interface__Basic_4 (114)
Basic
"] + + q2 --> q96 + q96 -->|"tok("interface")"| q97 + q97 -->|"tok(ID)"| q106 + q98 -->|"tok("extends")"| q99 + q99 -->|"tok(ID)"| q103 + q100 -->|"tok(",")"| q101 + q101 -->|"tok(ID)"| q102 + q102 --> q105 + q103 --> q100 + q103 --> q104 + q104 --> q107 + q105 --> q103 + q106 --> q98 + q106 --> q104 + q107 -->|"tok("{")"| q110 + q108 -.->|"[Field]"| q109 + q109 --> q112 + q110 --> q108 + q110 --> q111 + q111 --> q113 + q112 --> q110 + q113 -->|"tok("}")"| q114 + q114 --> q3 ``` ## Field @@ -113,14 +118,14 @@ flowchart TD flowchart TD q4(["Field__Start (4)
RuleStart"]) q5(["Field__Stop (5)
RuleStop"]) - q91["Field_Name_ID (91)
Basic
"] - q92["Field__Basic_0 (92)
Basic
"] - q93["Field__Basic_1 (93)
Basic
"] - - q4 --> q91 - q91 -->|"tok(ID)"| q92 - q92 -.->|"[FieldType]"| q93 - q93 --> q5 + q115["Field_Name_ID (115)
Basic
"] + q116["Field__Basic_0 (116)
Basic
"] + q117["Field__Basic_1 (117)
Basic
"] + + q4 --> q115 + q115 -->|"tok(ID)"| q116 + q116 -.->|"[FieldType]"| q117 + q117 --> q5 ``` ## FieldType @@ -129,31 +134,31 @@ flowchart TD flowchart TD q6(["FieldType__Start (6)
RuleStart"]) q7(["FieldType__Stop (7)
RuleStop"]) - q94["FieldType__Basic_0 (94)
Basic
"] - q95["FieldType__Basic_1 (95)
Basic
"] - q96["FieldType__Basic_2 (96)
Basic
"] - q97["FieldType__Basic_3 (97)
Basic
"] - q98["FieldType__Basic_4 (98)
Basic
"] - q99["FieldType__Basic_5 (99)
Basic
"] - q100["FieldType__Basic_6 (100)
Basic
"] - q101["FieldType__Basic_7 (101)
Basic
"] - q102{"FieldType__Basic_8 (102)
Basic

dec=6"} - q103["FieldType__BlockEnd (103)
BlockEnd
"] - - q6 --> q102 - q94 -.->|"[SimpleType]"| q95 - q95 --> q103 - q96 -.->|"[ReferenceType]"| q97 - q97 --> q103 - q98 -.->|"[ArrayType]"| q99 - q99 --> q103 - q100 -.->|"[PrimitiveType]"| q101 - q101 --> q103 - q102 --> q94 - q102 --> q96 - q102 --> q98 - q102 --> q100 - q103 --> q7 + q118["FieldType__Basic_0 (118)
Basic
"] + q119["FieldType__Basic_1 (119)
Basic
"] + q120["FieldType__Basic_2 (120)
Basic
"] + q121["FieldType__Basic_3 (121)
Basic
"] + q122["FieldType__Basic_4 (122)
Basic
"] + q123["FieldType__Basic_5 (123)
Basic
"] + q124["FieldType__Basic_6 (124)
Basic
"] + q125["FieldType__Basic_7 (125)
Basic
"] + q126{"FieldType__Basic_8 (126)
Basic

dec=6"} + q127["FieldType__BlockEnd (127)
BlockEnd
"] + + q6 --> q126 + q118 -.->|"[SimpleType]"| q119 + q119 --> q127 + q120 -.->|"[ReferenceType]"| q121 + q121 --> q127 + q122 -.->|"[ArrayType]"| q123 + q123 --> q127 + q124 -.->|"[PrimitiveType]"| q125 + q125 --> q127 + q126 --> q118 + q126 --> q120 + q126 --> q122 + q126 --> q124 + q127 --> q7 ``` ## ArrayType @@ -162,16 +167,16 @@ flowchart TD flowchart TD q8(["ArrayType__Start (8)
RuleStart"]) q9(["ArrayType__Stop (9)
RuleStop"]) - q104["ArrayType_LeftBracket (104)
Basic
"] - q105["ArrayType_RightBracket (105)
Basic
"] - q106["ArrayType__Basic_0 (106)
Basic
"] - q107["ArrayType__Basic_1 (107)
Basic
"] - - q8 --> q104 - q104 -->|"tok("[")"| q105 - q105 -->|"tok("]")"| q106 - q106 -.->|"[FieldType]"| q107 - q107 --> q9 + q128["ArrayType_LeftBracket (128)
Basic
"] + q129["ArrayType_RightBracket (129)
Basic
"] + q130["ArrayType__Basic_0 (130)
Basic
"] + q131["ArrayType__Basic_1 (131)
Basic
"] + + q8 --> q128 + q128 -->|"tok("[")"| q129 + q129 -->|"tok("]")"| q130 + q130 -.->|"[FieldType]"| q131 + q131 --> q9 ``` ## ReferenceType @@ -180,14 +185,14 @@ flowchart TD flowchart TD q10(["ReferenceType__Start (10)
RuleStart"]) q11(["ReferenceType__Stop (11)
RuleStop"]) - q108["ReferenceType_Asterisk (108)
Basic
"] - q109["ReferenceType_Type_ID (109)
Basic
"] - q110["ReferenceType__Basic (110)
Basic
"] - - q10 --> q108 - q108 -->|"tok("*")"| q109 - q109 -->|"tok(ID)"| q110 - q110 --> q11 + q132["ReferenceType_Asterisk (132)
Basic
"] + q133["ReferenceType_Type_ID (133)
Basic
"] + q134["ReferenceType__Basic (134)
Basic
"] + + q10 --> q132 + q132 -->|"tok("*")"| q133 + q133 -->|"tok(ID)"| q134 + q134 --> q11 ``` ## SimpleType @@ -196,12 +201,12 @@ flowchart TD flowchart TD q12(["SimpleType__Start (12)
RuleStart"]) q13(["SimpleType__Stop (13)
RuleStop"]) - q111["SimpleType_Type_ID (111)
Basic
"] - q112["SimpleType__Basic (112)
Basic
"] + q135["SimpleType_Type_ID (135)
Basic
"] + q136["SimpleType__Basic (136)
Basic
"] - q12 --> q111 - q111 -->|"tok(ID)"| q112 - q112 --> q13 + q12 --> q135 + q135 -->|"tok(ID)"| q136 + q136 --> q13 ``` ## PrimitiveType @@ -210,26 +215,26 @@ flowchart TD flowchart TD q14(["PrimitiveType__Start (14)
RuleStart"]) q15(["PrimitiveType__Stop (15)
RuleStop"]) - q113["PrimitiveType_Type_string (113)
Basic
"] - q114["PrimitiveType__Basic_0 (114)
Basic
"] - q115["PrimitiveType_Type_bool (115)
Basic
"] - q116["PrimitiveType__Basic_1 (116)
Basic
"] - q117["PrimitiveType_Type_composite (117)
Basic
"] - q118["PrimitiveType__Basic_2 (118)
Basic
"] - q119{"PrimitiveType__Basic_3 (119)
Basic

dec=7"} - q120["PrimitiveType__BlockEnd (120)
BlockEnd
"] - - q14 --> q119 - q113 -->|"tok("string")"| q114 - q114 --> q120 - q115 -->|"tok("bool")"| q116 - q116 --> q120 - q117 -->|"tok("composite")"| q118 - q118 --> q120 - q119 --> q113 - q119 --> q115 - q119 --> q117 - q120 --> q15 + q137["PrimitiveType_Type_string (137)
Basic
"] + q138["PrimitiveType__Basic_0 (138)
Basic
"] + q139["PrimitiveType_Type_bool (139)
Basic
"] + q140["PrimitiveType__Basic_1 (140)
Basic
"] + q141["PrimitiveType_Type_composite (141)
Basic
"] + q142["PrimitiveType__Basic_2 (142)
Basic
"] + q143{"PrimitiveType__Basic_3 (143)
Basic

dec=7"} + q144["PrimitiveType__BlockEnd (144)
BlockEnd
"] + + q14 --> q143 + q137 -->|"tok("string")"| q138 + q138 --> q144 + q139 -->|"tok("bool")"| q140 + q140 --> q144 + q141 -->|"tok("composite")"| q142 + q142 --> q144 + q143 --> q137 + q143 --> q139 + q143 --> q141 + q144 --> q15 ``` ## ParserRule @@ -238,589 +243,878 @@ flowchart TD flowchart TD q16(["ParserRule__Start (16)
RuleStart"]) q17(["ParserRule__Stop (17)
RuleStop"]) - q121["ParserRule_Entry_entry (121)
Basic
"] - q122["ParserRule__Basic_0 (122)
Basic
"] - q123{"ParserRule__Basic_1 (123)
Basic

dec=8"} - q124["ParserRule_Name_ID (124)
Basic
"] - q125["ParserRule_returns (125)
Basic
"] - q126["ParserRule_ReturnType_ID (126)
Basic
"] - q127["ParserRule__Basic_2 (127)
Basic
"] - q128{"ParserRule__Basic_3 (128)
Basic

dec=9"} - q129["ParserRule_Colon (129)
Basic
"] - q130["ParserRule__Basic_4 (130)
Basic
"] - q131["ParserRule_Semicolon (131)
Basic
"] - q132["ParserRule__Basic_5 (132)
Basic
"] - q133{"ParserRule__Basic_6 (133)
Basic

dec=10"} - - q16 --> q123 - q121 -->|"tok("entry")"| q122 - q122 --> q124 - q123 --> q121 - q123 --> q122 - q124 -->|"tok(ID)"| q128 - q125 -->|"tok("returns")"| q126 - q126 -->|"tok(ID)"| q127 - q127 --> q129 - q128 --> q125 - q128 --> q127 - q129 -->|"tok(":")"| q130 - q130 -.->|"[Alternatives]"| q133 - q131 -->|"tok(";")"| q132 - q132 --> q17 - q133 --> q131 - q133 --> q132 -``` - -## Token - -```mermaid -flowchart TD - q18(["Token__Start (18)
RuleStart"]) - q19(["Token__Stop (19)
RuleStop"]) - q134["Token_Type_hidden (134)
Basic
"] - q135["Token__Basic_0 (135)
Basic
"] - q136["Token_Type_comment (136)
Basic
"] - q137["Token__Basic_1 (137)
Basic
"] - q138{"Token__Basic_2 (138)
Basic

dec=11"} - q139["Token__BlockEnd (139)
BlockEnd
"] - q140{"Token__Basic_3 (140)
Basic

dec=12"} - q141["Token_token (141)
Basic
"] - q142["Token_Name_ID (142)
Basic
"] - q143["Token_Colon (143)
Basic
"] - q144["Token_Regexp_RegexLiteral (144)
Basic
"] - q145["Token_Semicolon (145)
Basic
"] - q146["Token__Basic_4 (146)
Basic
"] - q147{"Token__Basic_5 (147)
Basic

dec=13"} - - q18 --> q140 - q134 -->|"tok("hidden")"| q135 - q135 --> q139 - q136 -->|"tok("comment")"| q137 - q137 --> q139 - q138 --> q134 - q138 --> q136 - q139 --> q141 - q140 --> q138 - q140 --> q139 - q141 -->|"tok("token")"| q142 - q142 -->|"tok(ID)"| q143 - q143 -->|"tok(":")"| q144 - q144 -->|"tok(RegexLiteral)"| q147 - q145 -->|"tok(";")"| q146 - q146 --> q19 + q145["ParserRule_Entry_entry (145)
Basic
"] + q146["ParserRule__Basic_0 (146)
Basic
"] + q147{"ParserRule__Basic_1 (147)
Basic

dec=8"} + q148["ParserRule_Name_ID (148)
Basic
"] + q149["ParserRule_returns (149)
Basic
"] + q150["ParserRule_ReturnType_ID (150)
Basic
"] + q151["ParserRule__Basic_2 (151)
Basic
"] + q152{"ParserRule__Basic_3 (152)
Basic

dec=9"} + q153["ParserRule_Colon (153)
Basic
"] + q154["ParserRule__Basic_4 (154)
Basic
"] + q155["ParserRule_Semicolon (155)
Basic
"] + q156["ParserRule__Basic_5 (156)
Basic
"] + q157{"ParserRule__Basic_6 (157)
Basic

dec=10"} + + q16 --> q147 + q145 -->|"tok("entry")"| q146 + q146 --> q148 q147 --> q145 q147 --> q146 + q148 -->|"tok(ID)"| q152 + q149 -->|"tok("returns")"| q150 + q150 -->|"tok(ID)"| q151 + q151 --> q153 + q152 --> q149 + q152 --> q151 + q153 -->|"tok(":")"| q154 + q154 -.->|"[Alternatives]"| q157 + q155 -->|"tok(";")"| q156 + q156 --> q17 + q157 --> q155 + q157 --> q156 +``` + +## TokenDecl + +```mermaid +flowchart TD + q18(["TokenDecl__Start (18)
RuleStart"]) + q19(["TokenDecl__Stop (19)
RuleStop"]) + q158["TokenDecl__Basic_0 (158)
Basic
"] + q159["TokenDecl__Basic_1 (159)
Basic
"] + q160{"TokenDecl__Basic_2 (160)
Basic

dec=11"} + q161["TokenDecl_token (161)
Basic
"] + q162["TokenDecl_Name_ID (162)
Basic
"] + q163["TokenDecl_Colon (163)
Basic
"] + q164["TokenDecl__Basic_3 (164)
Basic
"] + q165["TokenDecl__Basic_4 (165)
Basic
"] + q166["TokenDecl__Basic_5 (166)
Basic
"] + q167{"TokenDecl__Basic_6 (167)
Basic

dec=12"} + q168["TokenDecl_Semicolon (168)
Basic
"] + q169["TokenDecl__Basic_7 (169)
Basic
"] + q170{"TokenDecl__Basic_8 (170)
Basic

dec=13"} + + q18 --> q160 + q158 -->|"tok(TokenModifier)"| q159 + q159 --> q161 + q160 --> q158 + q160 --> q159 + q161 -->|"tok("token")"| q162 + q162 -->|"tok(ID)"| q163 + q163 -->|"tok(":")"| q164 + q164 -.->|"[TokenContent]"| q167 + q165 -.->|"[TokenCommand]"| q166 + q166 --> q170 + q167 --> q165 + q167 --> q166 + q168 -->|"tok(";")"| q169 + q169 --> q19 + q170 --> q168 + q170 --> q169 +``` + +## TokenContent + +```mermaid +flowchart TD + q20(["TokenContent__Start (20)
RuleStart"]) + q21(["TokenContent__Stop (21)
RuleStop"]) + q171["TokenContent__Basic_0 (171)
Basic
"] + q172["TokenContent__Basic_1 (172)
Basic
"] + q173["TokenContent__Basic_2 (173)
Basic
"] + q174["TokenContent__Basic_3 (174)
Basic
"] + q175{"TokenContent__Basic_4 (175)
Basic

dec=14"} + q176["TokenContent__BlockEnd (176)
BlockEnd
"] + + q20 --> q175 + q171 -.->|"[RegexpTokenContent]"| q172 + q172 --> q176 + q173 -.->|"[KeywordTokenContent]"| q174 + q174 --> q176 + q175 --> q171 + q175 --> q173 + q176 --> q21 +``` + +## RegexpTokenContent + +```mermaid +flowchart TD + q22(["RegexpTokenContent__Start (22)
RuleStart"]) + q23(["RegexpTokenContent__Stop (23)
RuleStop"]) + q177["RegexpTokenContent_Regexp_RegexLiteral (177)
Basic
"] + q178["RegexpTokenContent__Basic (178)
Basic
"] + + q22 --> q177 + q177 -->|"tok(RegexLiteral)"| q178 + q178 --> q23 +``` + +## KeywordTokenContent + +```mermaid +flowchart TD + q24(["KeywordTokenContent__Start (24)
RuleStart"]) + q25(["KeywordTokenContent__Stop (25)
RuleStop"]) + q179["KeywordTokenContent__Basic_0 (179)
Basic
"] + q180["KeywordTokenContent__Basic_1 (180)
Basic
"] + + q24 --> q179 + q179 -.->|"[Keyword]"| q180 + q180 --> q25 +``` + +## TokenCommand + +```mermaid +flowchart TD + q26(["TokenCommand__Start (26)
RuleStart"]) + q27(["TokenCommand__Stop (27)
RuleStop"]) + q181["TokenCommand_DashGreaterThan (181)
Basic
"] + q182["TokenCommand_Type_push (182)
Basic
"] + q183["TokenCommand__Basic_0 (183)
Basic
"] + q184["TokenCommand_Type_pop (184)
Basic
"] + q185["TokenCommand__Basic_1 (185)
Basic
"] + q186["TokenCommand_Type_mode (186)
Basic
"] + q187["TokenCommand__Basic_2 (187)
Basic
"] + q188{"TokenCommand__Basic_3 (188)
Basic

dec=15"} + q189["TokenCommand__BlockEnd_0 (189)
BlockEnd
"] + q190["TokenCommand_LeftParen (190)
Basic
"] + q191["TokenCommand_Mode_ID (191)
Basic
"] + q192["TokenCommand__Basic_4 (192)
Basic
"] + q193["TokenCommand_Default_default (193)
Basic
"] + q194["TokenCommand__Basic_5 (194)
Basic
"] + q195{"TokenCommand__Basic_6 (195)
Basic

dec=16"} + q196["TokenCommand__BlockEnd_1 (196)
BlockEnd
"] + q197["TokenCommand_RightParen (197)
Basic
"] + q198["TokenCommand__Basic_7 (198)
Basic
"] + q199{"TokenCommand__Basic_8 (199)
Basic

dec=17"} + + q26 --> q181 + q181 -->|"tok("->")"| q188 + q182 -->|"tok("push")"| q183 + q183 --> q189 + q184 -->|"tok("pop")"| q185 + q185 --> q189 + q186 -->|"tok("mode")"| q187 + q187 --> q189 + q188 --> q182 + q188 --> q184 + q188 --> q186 + q189 --> q199 + q190 -->|"tok("(")"| q195 + q191 -->|"tok(ID)"| q192 + q192 --> q196 + q193 -->|"tok("default")"| q194 + q194 --> q196 + q195 --> q191 + q195 --> q193 + q196 --> q197 + q197 -->|"tok(")")"| q198 + q198 --> q27 + q199 --> q190 + q199 --> q198 ``` ## TokenGroup ```mermaid flowchart TD - q20(["TokenGroup__Start (20)
RuleStart"]) - q21(["TokenGroup__Stop (21)
RuleStop"]) - q148["TokenGroup_token (148)
Basic
"] - q149["TokenGroup_group (149)
Basic
"] - q150["TokenGroup_Name_ID (150)
Basic
"] - q151["TokenGroup_LeftBrace (151)
Basic
"] - q152["TokenGroup_TokenRefs_ID (152)
Basic
"] - q153["TokenGroup__Basic_0 (153)
Basic
"] - q154["TokenGroup_keywords (154)
Basic
"] - q155["TokenGroup_Regexps_RegexLiteral (155)
Basic
"] - q156["TokenGroup__Basic_1 (156)
Basic
"] - q157["TokenGroup__Basic_2 (157)
Basic
"] - q158["TokenGroup__Basic_3 (158)
Basic
"] - q159{"TokenGroup__Basic_4 (159)
Basic

dec=14"} - q160["TokenGroup__BlockEnd (160)
BlockEnd
"] - q161{"TokenGroup__LoopEntry (161)
LoopEntry

dec=15"} - q162["TokenGroup__LoopEnd (162)
LoopEnd
"] - q163["TokenGroup__LoopBack (163)
LoopBack
"] - q164["TokenGroup_RightBrace (164)
Basic
"] - q165["TokenGroup__Basic_5 (165)
Basic
"] - - q20 --> q148 - q148 -->|"tok("token")"| q149 - q149 -->|"tok("group")"| q150 - q150 -->|"tok(ID)"| q151 - q151 -->|"tok("{")"| q161 - q152 -->|"tok(ID)"| q153 - q153 --> q160 - q154 -->|"tok("keywords")"| q155 - q155 -->|"tok(RegexLiteral)"| q156 - q156 --> q160 - q157 -.->|"[Keyword]"| q158 - q158 --> q160 - q159 --> q152 - q159 --> q154 - q159 --> q157 - q160 --> q163 - q161 --> q159 - q161 --> q162 - q162 --> q164 - q163 --> q161 - q164 -->|"tok("}")"| q165 - q165 --> q21 + q28(["TokenGroup__Start (28)
RuleStart"]) + q29(["TokenGroup__Stop (29)
RuleStop"]) + q200["TokenGroup__Basic_0 (200)
Basic
"] + q201["TokenGroup__Basic_1 (201)
Basic
"] + q202{"TokenGroup__Basic_2 (202)
Basic

dec=18"} + q203["TokenGroup_token (203)
Basic
"] + q204["TokenGroup_group (204)
Basic
"] + q205["TokenGroup_Name_ID (205)
Basic
"] + q206["TokenGroup_LeftBrace (206)
Basic
"] + q207["TokenGroup_TokenRefs_ID (207)
Basic
"] + q208["TokenGroup__Basic_3 (208)
Basic
"] + q209["TokenGroup__Basic_4 (209)
Basic
"] + q210["TokenGroup__Basic_5 (210)
Basic
"] + q211["TokenGroup_keywords (211)
Basic
"] + q212["TokenGroup_KeywordSelectors_RegexLiteral (212)
Basic
"] + q213["TokenGroup__Basic_6 (213)
Basic
"] + q214{"TokenGroup__Basic_7 (214)
Basic

dec=19"} + q215["TokenGroup__BlockEnd (215)
BlockEnd
"] + q216{"TokenGroup__LoopEntry (216)
LoopEntry

dec=20"} + q217["TokenGroup__LoopEnd (217)
LoopEnd
"] + q218["TokenGroup__LoopBack (218)
LoopBack
"] + q219["TokenGroup_RightBrace (219)
Basic
"] + q220["TokenGroup__Basic_8 (220)
Basic
"] + q221["TokenGroup__Basic_9 (221)
Basic
"] + q222{"TokenGroup__Basic_10 (222)
Basic

dec=21"} + q223["TokenGroup_Semicolon (223)
Basic
"] + q224["TokenGroup__Basic_11 (224)
Basic
"] + q225{"TokenGroup__Basic_12 (225)
Basic

dec=22"} + + q28 --> q202 + q200 -->|"tok(TokenModifier)"| q201 + q201 --> q203 + q202 --> q200 + q202 --> q201 + q203 -->|"tok("token")"| q204 + q204 -->|"tok("group")"| q205 + q205 -->|"tok(ID)"| q206 + q206 -->|"tok("{")"| q216 + q207 -->|"tok(ID)"| q208 + q208 --> q215 + q209 -.->|"[Keyword]"| q210 + q210 --> q215 + q211 -->|"tok("keywords")"| q212 + q212 -->|"tok(RegexLiteral)"| q213 + q213 --> q215 + q214 --> q207 + q214 --> q209 + q214 --> q211 + q215 --> q218 + q216 --> q214 + q216 --> q217 + q217 --> q219 + q218 --> q216 + q219 -->|"tok("}")"| q222 + q220 -.->|"[TokenCommand]"| q221 + q221 --> q225 + q222 --> q220 + q222 --> q221 + q223 -->|"tok(";")"| q224 + q224 --> q29 + q225 --> q223 + q225 --> q224 ``` -## Alternatives +## TokenMode ```mermaid flowchart TD - q22(["Alternatives__Start (22)
RuleStart"]) - q23(["Alternatives__Stop (23)
RuleStop"]) - q166["Alternatives__Basic_0 (166)
Basic
"] - q167["Alternatives_Pipe (167)
Basic
"] - q168["Alternatives__Basic_1 (168)
Basic
"] - q169["Alternatives__Basic_2 (169)
Basic
"] - q170{"Alternatives__LoopBack (170)
LoopBack

dec=16"} - q171["Alternatives__LoopEnd (171)
LoopEnd
"] - q172{"Alternatives__Basic_3 (172)
Basic

dec=17"} + q30(["TokenMode__Start (30)
RuleStart"]) + q31(["TokenMode__Stop (31)
RuleStop"]) + q226["TokenMode_token (226)
Basic
"] + q227["TokenMode_mode (227)
Basic
"] + q228["TokenMode_Name_ID (228)
Basic
"] + q229["TokenMode__Basic_0 (229)
Basic
"] + q230["TokenMode_Default_default (230)
Basic
"] + q231["TokenMode__Basic_1 (231)
Basic
"] + q232{"TokenMode__Basic_2 (232)
Basic

dec=23"} + q233["TokenMode__BlockEnd (233)
BlockEnd
"] + q234["TokenMode_LeftBrace (234)
Basic
"] + q235["TokenMode__Basic_3 (235)
Basic
"] + q236["TokenMode__Basic_4 (236)
Basic
"] + q237{"TokenMode__LoopEntry (237)
LoopEntry

dec=24"} + q238["TokenMode__LoopEnd (238)
LoopEnd
"] + q239["TokenMode__LoopBack (239)
LoopBack
"] + q240["TokenMode_RightBrace (240)
Basic
"] + q241["TokenMode__Basic_5 (241)
Basic
"] + + q30 --> q226 + q226 -->|"tok("token")"| q227 + q227 -->|"tok("mode")"| q232 + q228 -->|"tok(ID)"| q229 + q229 --> q233 + q230 -->|"tok("default")"| q231 + q231 --> q233 + q232 --> q228 + q232 --> q230 + q233 --> q234 + q234 -->|"tok("{")"| q237 + q235 -.->|"[TokenModeMember]"| q236 + q236 --> q239 + q237 --> q235 + q237 --> q238 + q238 --> q240 + q239 --> q237 + q240 -->|"tok("}")"| q241 + q241 --> q31 +``` + +## TokenModeMember - q22 --> q166 - q166 -.->|"[Group]"| q172 - q167 -->|"tok("|")"| q168 - q168 -.->|"[Group]"| q169 - q169 --> q170 - q170 --> q167 - q170 --> q171 - q171 --> q23 - q172 --> q167 - q172 --> q171 +```mermaid +flowchart TD + q32(["TokenModeMember__Start (32)
RuleStart"]) + q33(["TokenModeMember__Stop (33)
RuleStop"]) + q242["TokenModeMember__Basic_0 (242)
Basic
"] + q243["TokenModeMember__Basic_1 (243)
Basic
"] + q244["TokenModeMember__Basic_2 (244)
Basic
"] + q245["TokenModeMember__Basic_3 (245)
Basic
"] + q246["TokenModeMember__Basic_4 (246)
Basic
"] + q247["TokenModeMember__Basic_5 (247)
Basic
"] + q248["TokenModeMember__Basic_6 (248)
Basic
"] + q249["TokenModeMember__Basic_7 (249)
Basic
"] + q250["TokenModeMember__Basic_8 (250)
Basic
"] + q251["TokenModeMember__Basic_9 (251)
Basic
"] + q252{"TokenModeMember__Basic_10 (252)
Basic

dec=25"} + q253["TokenModeMember__BlockEnd (253)
BlockEnd
"] + + q32 --> q252 + q242 -.->|"[TokenDeclUsage]"| q243 + q243 --> q253 + q244 -.->|"[TokenGroupUsage]"| q245 + q245 --> q253 + q246 -.->|"[TokenUsage]"| q247 + q247 --> q253 + q248 -.->|"[KeywordUsage]"| q249 + q249 --> q253 + q250 -.->|"[KeywordSelector]"| q251 + q251 --> q253 + q252 --> q242 + q252 --> q244 + q252 --> q246 + q252 --> q248 + q252 --> q250 + q253 --> q33 ``` -## Group +## TokenDeclUsage ```mermaid flowchart TD - q24(["Group__Start (24)
RuleStart"]) - q25(["Group__Stop (25)
RuleStop"]) - q173["Group__Basic_0 (173)
Basic
"] - q174["Group__Basic_1 (174)
Basic
"] - q175["Group__Basic_2 (175)
Basic
"] - q176{"Group__LoopBack (176)
LoopBack

dec=18"} - q177["Group__LoopEnd (177)
LoopEnd
"] - q178{"Group__Basic_3 (178)
Basic

dec=19"} + q34(["TokenDeclUsage__Start (34)
RuleStart"]) + q35(["TokenDeclUsage__Stop (35)
RuleStop"]) + q254["TokenDeclUsage__Basic_0 (254)
Basic
"] + q255["TokenDeclUsage__Basic_1 (255)
Basic
"] + + q34 --> q254 + q254 -.->|"[TokenDecl]"| q255 + q255 --> q35 +``` + +## TokenGroupUsage - q24 --> q173 - q173 -.->|"[Element]"| q178 - q174 -.->|"[Element]"| q175 - q175 --> q176 - q176 --> q174 - q176 --> q177 - q177 --> q25 - q178 --> q174 - q178 --> q177 +```mermaid +flowchart TD + q36(["TokenGroupUsage__Start (36)
RuleStart"]) + q37(["TokenGroupUsage__Stop (37)
RuleStop"]) + q256["TokenGroupUsage__Basic_0 (256)
Basic
"] + q257["TokenGroupUsage__Basic_1 (257)
Basic
"] + + q36 --> q256 + q256 -.->|"[TokenGroup]"| q257 + q257 --> q37 +``` + +## TokenUsage + +```mermaid +flowchart TD + q38(["TokenUsage__Start (38)
RuleStart"]) + q39(["TokenUsage__Stop (39)
RuleStop"]) + q258["TokenUsage__Basic_0 (258)
Basic
"] + q259["TokenUsage__Basic_1 (259)
Basic
"] + q260{"TokenUsage__Basic_2 (260)
Basic

dec=26"} + q261["TokenUsage_TokenRef_ID (261)
Basic
"] + q262["TokenUsage__Basic_3 (262)
Basic
"] + q263["TokenUsage__Basic_4 (263)
Basic
"] + q264{"TokenUsage__Basic_5 (264)
Basic

dec=27"} + q265["TokenUsage_Semicolon (265)
Basic
"] + q266["TokenUsage__Basic_6 (266)
Basic
"] + q267{"TokenUsage__Basic_7 (267)
Basic

dec=28"} + + q38 --> q260 + q258 -->|"tok(TokenModifier)"| q259 + q259 --> q261 + q260 --> q258 + q260 --> q259 + q261 -->|"tok(ID)"| q264 + q262 -.->|"[TokenCommand]"| q263 + q263 --> q267 + q264 --> q262 + q264 --> q263 + q265 -->|"tok(";")"| q266 + q266 --> q39 + q267 --> q265 + q267 --> q266 +``` + +## KeywordUsage + +```mermaid +flowchart TD + q40(["KeywordUsage__Start (40)
RuleStart"]) + q41(["KeywordUsage__Stop (41)
RuleStop"]) + q268["KeywordUsage__Basic_0 (268)
Basic
"] + q269["KeywordUsage__Basic_1 (269)
Basic
"] + q270{"KeywordUsage__Basic_2 (270)
Basic

dec=29"} + q271["KeywordUsage__Basic_3 (271)
Basic
"] + q272["KeywordUsage__Basic_4 (272)
Basic
"] + q273["KeywordUsage__Basic_5 (273)
Basic
"] + q274{"KeywordUsage__Basic_6 (274)
Basic

dec=30"} + q275["KeywordUsage_Semicolon (275)
Basic
"] + q276["KeywordUsage__Basic_7 (276)
Basic
"] + q277{"KeywordUsage__Basic_8 (277)
Basic

dec=31"} + + q40 --> q270 + q268 -->|"tok(TokenModifier)"| q269 + q269 --> q271 + q270 --> q268 + q270 --> q269 + q271 -.->|"[Keyword]"| q274 + q272 -.->|"[TokenCommand]"| q273 + q273 --> q277 + q274 --> q272 + q274 --> q273 + q275 -->|"tok(";")"| q276 + q276 --> q41 + q277 --> q275 + q277 --> q276 +``` + +## KeywordSelector + +```mermaid +flowchart TD + q42(["KeywordSelector__Start (42)
RuleStart"]) + q43(["KeywordSelector__Stop (43)
RuleStop"]) + q278["KeywordSelector_keywords (278)
Basic
"] + q279["KeywordSelector_Selector_RegexLiteral (279)
Basic
"] + q280["KeywordSelector_Semicolon (280)
Basic
"] + q281["KeywordSelector__Basic_0 (281)
Basic
"] + q282{"KeywordSelector__Basic_1 (282)
Basic

dec=32"} + + q42 --> q278 + q278 -->|"tok("keywords")"| q279 + q279 -->|"tok(RegexLiteral)"| q282 + q280 -->|"tok(";")"| q281 + q281 --> q43 + q282 --> q280 + q282 --> q281 +``` + +## Alternatives + +```mermaid +flowchart TD + q44(["Alternatives__Start (44)
RuleStart"]) + q45(["Alternatives__Stop (45)
RuleStop"]) + q283["Alternatives__Basic_0 (283)
Basic
"] + q284["Alternatives_Pipe (284)
Basic
"] + q285["Alternatives__Basic_1 (285)
Basic
"] + q286["Alternatives__Basic_2 (286)
Basic
"] + q287{"Alternatives__LoopBack (287)
LoopBack

dec=33"} + q288["Alternatives__LoopEnd (288)
LoopEnd
"] + q289{"Alternatives__Basic_3 (289)
Basic

dec=34"} + + q44 --> q283 + q283 -.->|"[Group]"| q289 + q284 -->|"tok("|")"| q285 + q285 -.->|"[Group]"| q286 + q286 --> q287 + q287 --> q284 + q287 --> q288 + q288 --> q45 + q289 --> q284 + q289 --> q288 +``` + +## Group + +```mermaid +flowchart TD + q46(["Group__Start (46)
RuleStart"]) + q47(["Group__Stop (47)
RuleStop"]) + q290["Group__Basic_0 (290)
Basic
"] + q291["Group__Basic_1 (291)
Basic
"] + q292["Group__Basic_2 (292)
Basic
"] + q293{"Group__LoopBack (293)
LoopBack

dec=35"} + q294["Group__LoopEnd (294)
LoopEnd
"] + q295{"Group__Basic_3 (295)
Basic

dec=36"} + + q46 --> q290 + q290 -.->|"[Element]"| q295 + q291 -.->|"[Element]"| q292 + q292 --> q293 + q293 --> q291 + q293 --> q294 + q294 --> q47 + q295 --> q291 + q295 --> q294 ``` ## Element ```mermaid flowchart TD - q26(["Element__Start (26)
RuleStart"]) - q27(["Element__Stop (27)
RuleStop"]) - q179["Element__Basic_0 (179)
Basic
"] - q180["Element__Basic_1 (180)
Basic
"] - q181["Element__Basic_2 (181)
Basic
"] - q182["Element__Basic_3 (182)
Basic
"] - q183["Element__Basic_4 (183)
Basic
"] - q184["Element__Basic_5 (184)
Basic
"] - q185["Element__Basic_6 (185)
Basic
"] - q186["Element__Basic_7 (186)
Basic
"] - q187["Element_LeftParen (187)
Basic
"] - q188["Element__Basic_8 (188)
Basic
"] - q189["Element_RightParen (189)
Basic
"] - q190["Element__Basic_9 (190)
Basic
"] - q191{"Element__Basic_10 (191)
Basic

dec=20"} - q192["Element__BlockEnd_0 (192)
BlockEnd
"] - q193["Element_Cardinality_Asterisk (193)
Basic
"] - q194["Element__Basic_11 (194)
Basic
"] - q195["Element_Cardinality_Plus (195)
Basic
"] - q196["Element__Basic_12 (196)
Basic
"] - q197["Element_Cardinality_Question (197)
Basic
"] - q198["Element__Basic_13 (198)
Basic
"] - q199{"Element__Basic_14 (199)
Basic

dec=21"} - q200["Element__BlockEnd_1 (200)
BlockEnd
"] - q201{"Element__Basic_15 (201)
Basic

dec=22"} - - q26 --> q191 - q179 -.->|"[Keyword]"| q180 - q180 --> q192 - q181 -.->|"[Assignment]"| q182 - q182 --> q192 - q183 -.->|"[RuleCall]"| q184 - q184 --> q192 - q185 -.->|"[Action]"| q186 - q186 --> q192 - q187 -->|"tok("(")"| q188 - q188 -.->|"[Alternatives]"| q189 - q189 -->|"tok(")")"| q190 - q190 --> q192 - q191 --> q179 - q191 --> q181 - q191 --> q183 - q191 --> q185 - q191 --> q187 - q192 --> q201 - q193 -->|"tok("*")"| q194 - q194 --> q200 - q195 -->|"tok("+")"| q196 - q196 --> q200 - q197 -->|"tok("?")"| q198 - q198 --> q200 - q199 --> q193 - q199 --> q195 - q199 --> q197 - q200 --> q27 - q201 --> q199 - q201 --> q200 + q48(["Element__Start (48)
RuleStart"]) + q49(["Element__Stop (49)
RuleStop"]) + q296["Element__Basic_0 (296)
Basic
"] + q297["Element__Basic_1 (297)
Basic
"] + q298["Element__Basic_2 (298)
Basic
"] + q299["Element__Basic_3 (299)
Basic
"] + q300["Element__Basic_4 (300)
Basic
"] + q301["Element__Basic_5 (301)
Basic
"] + q302["Element__Basic_6 (302)
Basic
"] + q303["Element__Basic_7 (303)
Basic
"] + q304["Element_LeftParen (304)
Basic
"] + q305["Element__Basic_8 (305)
Basic
"] + q306["Element_RightParen (306)
Basic
"] + q307["Element__Basic_9 (307)
Basic
"] + q308{"Element__Basic_10 (308)
Basic

dec=37"} + q309["Element__BlockEnd (309)
BlockEnd
"] + q310["Element__Basic_11 (310)
Basic
"] + q311["Element__Basic_12 (311)
Basic
"] + q312{"Element__Basic_13 (312)
Basic

dec=38"} + + q48 --> q308 + q296 -.->|"[Keyword]"| q297 + q297 --> q309 + q298 -.->|"[Assignment]"| q299 + q299 --> q309 + q300 -.->|"[RuleCall]"| q301 + q301 --> q309 + q302 -.->|"[Action]"| q303 + q303 --> q309 + q304 -->|"tok("(")"| q305 + q305 -.->|"[Alternatives]"| q306 + q306 -->|"tok(")")"| q307 + q307 --> q309 + q308 --> q296 + q308 --> q298 + q308 --> q300 + q308 --> q302 + q308 --> q304 + q309 --> q312 + q310 -->|"tok(Cardinality)"| q311 + q311 --> q49 + q312 --> q310 + q312 --> q311 ``` ## Keyword ```mermaid flowchart TD - q28(["Keyword__Start (28)
RuleStart"]) - q29(["Keyword__Stop (29)
RuleStop"]) - q202["Keyword_Value_StringLiteral (202)
Basic
"] - q203["Keyword__Basic (203)
Basic
"] - - q28 --> q202 - q202 -->|"tok(StringLiteral)"| q203 - q203 --> q29 + q50(["Keyword__Start (50)
RuleStart"]) + q51(["Keyword__Stop (51)
RuleStop"]) + q313["Keyword_Value_StringLiteral (313)
Basic
"] + q314["Keyword__Basic (314)
Basic
"] + + q50 --> q313 + q313 -->|"tok(StringLiteral)"| q314 + q314 --> q51 ``` ## Assignment ```mermaid flowchart TD - q30(["Assignment__Start (30)
RuleStart"]) - q31(["Assignment__Stop (31)
RuleStop"]) - q204["Assignment_Property_ID (204)
Basic
"] - q205["Assignment_Operator_PlusEquals (205)
Basic
"] - q206["Assignment__Basic_0 (206)
Basic
"] - q207["Assignment_Operator_Equals (207)
Basic
"] - q208["Assignment__Basic_1 (208)
Basic
"] - q209["Assignment_Operator_QuestionEquals (209)
Basic
"] - q210["Assignment__Basic_2 (210)
Basic
"] - q211{"Assignment__Basic_3 (211)
Basic

dec=23"} - q212["Assignment__BlockEnd (212)
BlockEnd
"] - q213["Assignment__Basic_4 (213)
Basic
"] - q214["Assignment__Basic_5 (214)
Basic
"] - - q30 --> q204 - q204 -->|"tok(ID)"| q211 - q205 -->|"tok("+=")"| q206 - q206 --> q212 - q207 -->|"tok("=")"| q208 - q208 --> q212 - q209 -->|"tok("?=")"| q210 - q210 --> q212 - q211 --> q205 - q211 --> q207 - q211 --> q209 - q212 --> q213 - q213 -.->|"[Assignable]"| q214 - q214 --> q31 + q52(["Assignment__Start (52)
RuleStart"]) + q53(["Assignment__Stop (53)
RuleStop"]) + q315["Assignment_Property_ID (315)
Basic
"] + q316["Assignment_Operator_PlusEquals (316)
Basic
"] + q317["Assignment__Basic_0 (317)
Basic
"] + q318["Assignment_Operator_Equals (318)
Basic
"] + q319["Assignment__Basic_1 (319)
Basic
"] + q320["Assignment_Operator_QuestionEquals (320)
Basic
"] + q321["Assignment__Basic_2 (321)
Basic
"] + q322{"Assignment__Basic_3 (322)
Basic

dec=39"} + q323["Assignment__BlockEnd (323)
BlockEnd
"] + q324["Assignment__Basic_4 (324)
Basic
"] + q325["Assignment__Basic_5 (325)
Basic
"] + + q52 --> q315 + q315 -->|"tok(ID)"| q322 + q316 -->|"tok("+=")"| q317 + q317 --> q323 + q318 -->|"tok("=")"| q319 + q319 --> q323 + q320 -->|"tok("?=")"| q321 + q321 --> q323 + q322 --> q316 + q322 --> q318 + q322 --> q320 + q323 --> q324 + q324 -.->|"[Assignable]"| q325 + q325 --> q53 ``` ## Assignable ```mermaid flowchart TD - q32(["Assignable__Start (32)
RuleStart"]) - q33(["Assignable__Stop (33)
RuleStop"]) - q215["Assignable__Basic_0 (215)
Basic
"] - q216["Assignable__Basic_1 (216)
Basic
"] - q217["Assignable__Basic_2 (217)
Basic
"] - q218["Assignable__Basic_3 (218)
Basic
"] - q219["Assignable__Basic_4 (219)
Basic
"] - q220["Assignable__Basic_5 (220)
Basic
"] - q221["Assignable_LeftParen (221)
Basic
"] - q222["Assignable__Basic_6 (222)
Basic
"] - q223["Assignable_RightParen (223)
Basic
"] - q224["Assignable__Basic_7 (224)
Basic
"] - q225{"Assignable__Basic_8 (225)
Basic

dec=24"} - q226["Assignable__BlockEnd (226)
BlockEnd
"] - - q32 --> q225 - q215 -.->|"[Keyword]"| q216 - q216 --> q226 - q217 -.->|"[RuleCall]"| q218 - q218 --> q226 - q219 -.->|"[CrossRef]"| q220 - q220 --> q226 - q221 -->|"tok("(")"| q222 - q222 -.->|"[AssignableAlternatives]"| q223 - q223 -->|"tok(")")"| q224 - q224 --> q226 - q225 --> q215 - q225 --> q217 - q225 --> q219 - q225 --> q221 - q226 --> q33 + q54(["Assignable__Start (54)
RuleStart"]) + q55(["Assignable__Stop (55)
RuleStop"]) + q326["Assignable__Basic_0 (326)
Basic
"] + q327["Assignable__Basic_1 (327)
Basic
"] + q328["Assignable__Basic_2 (328)
Basic
"] + q329["Assignable__Basic_3 (329)
Basic
"] + q330["Assignable__Basic_4 (330)
Basic
"] + q331["Assignable__Basic_5 (331)
Basic
"] + q332["Assignable_LeftParen (332)
Basic
"] + q333["Assignable__Basic_6 (333)
Basic
"] + q334["Assignable_RightParen (334)
Basic
"] + q335["Assignable__Basic_7 (335)
Basic
"] + q336{"Assignable__Basic_8 (336)
Basic

dec=40"} + q337["Assignable__BlockEnd (337)
BlockEnd
"] + + q54 --> q336 + q326 -.->|"[Keyword]"| q327 + q327 --> q337 + q328 -.->|"[RuleCall]"| q329 + q329 --> q337 + q330 -.->|"[CrossRef]"| q331 + q331 --> q337 + q332 -->|"tok("(")"| q333 + q333 -.->|"[AssignableAlternatives]"| q334 + q334 -->|"tok(")")"| q335 + q335 --> q337 + q336 --> q326 + q336 --> q328 + q336 --> q330 + q336 --> q332 + q337 --> q55 ``` ## AssignableWithoutAlts ```mermaid flowchart TD - q34(["AssignableWithoutAlts__Start (34)
RuleStart"]) - q35(["AssignableWithoutAlts__Stop (35)
RuleStop"]) - q227["AssignableWithoutAlts__Basic_0 (227)
Basic
"] - q228["AssignableWithoutAlts__Basic_1 (228)
Basic
"] - q229["AssignableWithoutAlts__Basic_2 (229)
Basic
"] - q230["AssignableWithoutAlts__Basic_3 (230)
Basic
"] - q231["AssignableWithoutAlts__Basic_4 (231)
Basic
"] - q232["AssignableWithoutAlts__Basic_5 (232)
Basic
"] - q233{"AssignableWithoutAlts__Basic_6 (233)
Basic

dec=25"} - q234["AssignableWithoutAlts__BlockEnd (234)
BlockEnd
"] - - q34 --> q233 - q227 -.->|"[Keyword]"| q228 - q228 --> q234 - q229 -.->|"[RuleCall]"| q230 - q230 --> q234 - q231 -.->|"[CrossRef]"| q232 - q232 --> q234 - q233 --> q227 - q233 --> q229 - q233 --> q231 - q234 --> q35 + q56(["AssignableWithoutAlts__Start (56)
RuleStart"]) + q57(["AssignableWithoutAlts__Stop (57)
RuleStop"]) + q338["AssignableWithoutAlts__Basic_0 (338)
Basic
"] + q339["AssignableWithoutAlts__Basic_1 (339)
Basic
"] + q340["AssignableWithoutAlts__Basic_2 (340)
Basic
"] + q341["AssignableWithoutAlts__Basic_3 (341)
Basic
"] + q342["AssignableWithoutAlts__Basic_4 (342)
Basic
"] + q343["AssignableWithoutAlts__Basic_5 (343)
Basic
"] + q344{"AssignableWithoutAlts__Basic_6 (344)
Basic

dec=41"} + q345["AssignableWithoutAlts__BlockEnd (345)
BlockEnd
"] + + q56 --> q344 + q338 -.->|"[Keyword]"| q339 + q339 --> q345 + q340 -.->|"[RuleCall]"| q341 + q341 --> q345 + q342 -.->|"[CrossRef]"| q343 + q343 --> q345 + q344 --> q338 + q344 --> q340 + q344 --> q342 + q345 --> q57 ``` ## AssignableAlternatives ```mermaid flowchart TD - q36(["AssignableAlternatives__Start (36)
RuleStart"]) - q37(["AssignableAlternatives__Stop (37)
RuleStop"]) - q235["AssignableAlternatives__Basic_0 (235)
Basic
"] - q236["AssignableAlternatives_Pipe (236)
Basic
"] - q237["AssignableAlternatives__Basic_1 (237)
Basic
"] - q238["AssignableAlternatives__Basic_2 (238)
Basic
"] - q239{"AssignableAlternatives__LoopBack (239)
LoopBack

dec=26"} - q240["AssignableAlternatives__LoopEnd (240)
LoopEnd
"] - q241{"AssignableAlternatives__Basic_3 (241)
Basic

dec=27"} - - q36 --> q235 - q235 -.->|"[AssignableWithoutAlts]"| q241 - q236 -->|"tok("|")"| q237 - q237 -.->|"[AssignableWithoutAlts]"| q238 - q238 --> q239 - q239 --> q236 - q239 --> q240 - q240 --> q37 - q241 --> q236 - q241 --> q240 + q58(["AssignableAlternatives__Start (58)
RuleStart"]) + q59(["AssignableAlternatives__Stop (59)
RuleStop"]) + q346["AssignableAlternatives__Basic_0 (346)
Basic
"] + q347["AssignableAlternatives_Pipe (347)
Basic
"] + q348["AssignableAlternatives__Basic_1 (348)
Basic
"] + q349["AssignableAlternatives__Basic_2 (349)
Basic
"] + q350{"AssignableAlternatives__LoopBack (350)
LoopBack

dec=42"} + q351["AssignableAlternatives__LoopEnd (351)
LoopEnd
"] + q352{"AssignableAlternatives__Basic_3 (352)
Basic

dec=43"} + + q58 --> q346 + q346 -.->|"[AssignableWithoutAlts]"| q352 + q347 -->|"tok("|")"| q348 + q348 -.->|"[AssignableWithoutAlts]"| q349 + q349 --> q350 + q350 --> q347 + q350 --> q351 + q351 --> q59 + q352 --> q347 + q352 --> q351 ``` ## CrossRef ```mermaid flowchart TD - q38(["CrossRef__Start (38)
RuleStart"]) - q39(["CrossRef__Stop (39)
RuleStop"]) - q242["CrossRef_LeftBracket (242)
Basic
"] - q243["CrossRef_Type_ID (243)
Basic
"] - q244["CrossRef_Colon (244)
Basic
"] - q245["CrossRef__Basic_0 (245)
Basic
"] - q246["CrossRef__Basic_1 (246)
Basic
"] - q247{"CrossRef__Basic_2 (247)
Basic

dec=28"} - q248["CrossRef_RightBracket (248)
Basic
"] - q249["CrossRef__Basic_3 (249)
Basic
"] - - q38 --> q242 - q242 -->|"tok("[")"| q243 - q243 -->|"tok(ID)"| q247 - q244 -->|"tok(":")"| q245 - q245 -.->|"[RuleCall]"| q246 - q246 --> q248 - q247 --> q244 - q247 --> q246 - q248 -->|"tok("]")"| q249 - q249 --> q39 + q60(["CrossRef__Start (60)
RuleStart"]) + q61(["CrossRef__Stop (61)
RuleStop"]) + q353["CrossRef_LeftBracket (353)
Basic
"] + q354["CrossRef_Type_ID (354)
Basic
"] + q355["CrossRef_Colon (355)
Basic
"] + q356["CrossRef__Basic_0 (356)
Basic
"] + q357["CrossRef__Basic_1 (357)
Basic
"] + q358{"CrossRef__Basic_2 (358)
Basic

dec=44"} + q359["CrossRef_RightBracket (359)
Basic
"] + q360["CrossRef__Basic_3 (360)
Basic
"] + + q60 --> q353 + q353 -->|"tok("[")"| q354 + q354 -->|"tok(ID)"| q358 + q355 -->|"tok(":")"| q356 + q356 -.->|"[RuleCall]"| q357 + q357 --> q359 + q358 --> q355 + q358 --> q357 + q359 -->|"tok("]")"| q360 + q360 --> q61 ``` ## RuleCall ```mermaid flowchart TD - q40(["RuleCall__Start (40)
RuleStart"]) - q41(["RuleCall__Stop (41)
RuleStop"]) - q250["RuleCall_Rule_ID (250)
Basic
"] - q251["RuleCall__Basic (251)
Basic
"] - - q40 --> q250 - q250 -->|"tok(ID)"| q251 - q251 --> q41 + q62(["RuleCall__Start (62)
RuleStart"]) + q63(["RuleCall__Stop (63)
RuleStop"]) + q361["RuleCall_Rule_ID (361)
Basic
"] + q362["RuleCall__Basic (362)
Basic
"] + + q62 --> q361 + q361 -->|"tok(ID)"| q362 + q362 --> q63 ``` ## Action ```mermaid flowchart TD - q42(["Action__Start (42)
RuleStart"]) - q43(["Action__Stop (43)
RuleStop"]) - q252["Action_LeftBrace (252)
Basic
"] - q253["Action_Type_ID (253)
Basic
"] - q254["Action_Dot (254)
Basic
"] - q255["Action_Property_ID (255)
Basic
"] - q256["Action_Operator_PlusEquals (256)
Basic
"] - q257["Action__Basic_0 (257)
Basic
"] - q258["Action_Operator_Equals (258)
Basic
"] - q259["Action__Basic_1 (259)
Basic
"] - q260{"Action__Basic_2 (260)
Basic

dec=29"} - q261["Action__BlockEnd (261)
BlockEnd
"] - q262["Action_current (262)
Basic
"] - q263["Action__Basic_3 (263)
Basic
"] - q264{"Action__Basic_4 (264)
Basic

dec=30"} - q265["Action_RightBrace (265)
Basic
"] - q266["Action__Basic_5 (266)
Basic
"] - - q42 --> q252 - q252 -->|"tok("{")"| q253 - q253 -->|"tok(ID)"| q264 - q254 -->|"tok(".")"| q255 - q255 -->|"tok(ID)"| q260 - q256 -->|"tok("+=")"| q257 - q257 --> q261 - q258 -->|"tok("=")"| q259 - q259 --> q261 - q260 --> q256 - q260 --> q258 - q261 --> q262 - q262 -->|"tok("current")"| q263 - q263 --> q265 - q264 --> q254 - q264 --> q263 - q265 -->|"tok("}")"| q266 - q266 --> q43 + q64(["Action__Start (64)
RuleStart"]) + q65(["Action__Stop (65)
RuleStop"]) + q363["Action_LeftBrace (363)
Basic
"] + q364["Action_Type_ID (364)
Basic
"] + q365["Action_Dot (365)
Basic
"] + q366["Action_Property_ID (366)
Basic
"] + q367["Action_Operator_PlusEquals (367)
Basic
"] + q368["Action__Basic_0 (368)
Basic
"] + q369["Action_Operator_Equals (369)
Basic
"] + q370["Action__Basic_1 (370)
Basic
"] + q371{"Action__Basic_2 (371)
Basic

dec=45"} + q372["Action__BlockEnd (372)
BlockEnd
"] + q373["Action_current (373)
Basic
"] + q374["Action__Basic_3 (374)
Basic
"] + q375{"Action__Basic_4 (375)
Basic

dec=46"} + q376["Action_RightBrace (376)
Basic
"] + q377["Action__Basic_5 (377)
Basic
"] + + q64 --> q363 + q363 -->|"tok("{")"| q364 + q364 -->|"tok(ID)"| q375 + q365 -->|"tok(".")"| q366 + q366 -->|"tok(ID)"| q371 + q367 -->|"tok("+=")"| q368 + q368 --> q372 + q369 -->|"tok("=")"| q370 + q370 --> q372 + q371 --> q367 + q371 --> q369 + q372 --> q373 + q373 -->|"tok("current")"| q374 + q374 --> q376 + q375 --> q365 + q375 --> q374 + q376 -->|"tok("}")"| q377 + q377 --> q65 ``` ## CompositeRule ```mermaid flowchart TD - q44(["CompositeRule__Start (44)
RuleStart"]) - q45(["CompositeRule__Stop (45)
RuleStop"]) - q267["CompositeRule_composite (267)
Basic
"] - q268["CompositeRule_Name_ID (268)
Basic
"] - q269["CompositeRule_Colon (269)
Basic
"] - q270["CompositeRule__Basic_0 (270)
Basic
"] - q271["CompositeRule_Semicolon (271)
Basic
"] - q272["CompositeRule__Basic_1 (272)
Basic
"] - q273{"CompositeRule__Basic_2 (273)
Basic

dec=31"} - - q44 --> q267 - q267 -->|"tok("composite")"| q268 - q268 -->|"tok(ID)"| q269 - q269 -->|"tok(":")"| q270 - q270 -.->|"[CompositeAlternatives]"| q273 - q271 -->|"tok(";")"| q272 - q272 --> q45 - q273 --> q271 - q273 --> q272 + q66(["CompositeRule__Start (66)
RuleStart"]) + q67(["CompositeRule__Stop (67)
RuleStop"]) + q378["CompositeRule_composite (378)
Basic
"] + q379["CompositeRule_Name_ID (379)
Basic
"] + q380["CompositeRule_Colon (380)
Basic
"] + q381["CompositeRule__Basic_0 (381)
Basic
"] + q382["CompositeRule_Semicolon (382)
Basic
"] + q383["CompositeRule__Basic_1 (383)
Basic
"] + q384{"CompositeRule__Basic_2 (384)
Basic

dec=47"} + + q66 --> q378 + q378 -->|"tok("composite")"| q379 + q379 -->|"tok(ID)"| q380 + q380 -->|"tok(":")"| q381 + q381 -.->|"[CompositeAlternatives]"| q384 + q382 -->|"tok(";")"| q383 + q383 --> q67 + q384 --> q382 + q384 --> q383 ``` ## CompositeAlternatives ```mermaid flowchart TD - q46(["CompositeAlternatives__Start (46)
RuleStart"]) - q47(["CompositeAlternatives__Stop (47)
RuleStop"]) - q274["CompositeAlternatives__Basic_0 (274)
Basic
"] - q275["CompositeAlternatives_Pipe (275)
Basic
"] - q276["CompositeAlternatives__Basic_1 (276)
Basic
"] - q277["CompositeAlternatives__Basic_2 (277)
Basic
"] - q278{"CompositeAlternatives__LoopBack (278)
LoopBack

dec=32"} - q279["CompositeAlternatives__LoopEnd (279)
LoopEnd
"] - q280{"CompositeAlternatives__Basic_3 (280)
Basic

dec=33"} - - q46 --> q274 - q274 -.->|"[CompositeGroup]"| q280 - q275 -->|"tok("|")"| q276 - q276 -.->|"[CompositeGroup]"| q277 - q277 --> q278 - q278 --> q275 - q278 --> q279 - q279 --> q47 - q280 --> q275 - q280 --> q279 + q68(["CompositeAlternatives__Start (68)
RuleStart"]) + q69(["CompositeAlternatives__Stop (69)
RuleStop"]) + q385["CompositeAlternatives__Basic_0 (385)
Basic
"] + q386["CompositeAlternatives_Pipe (386)
Basic
"] + q387["CompositeAlternatives__Basic_1 (387)
Basic
"] + q388["CompositeAlternatives__Basic_2 (388)
Basic
"] + q389{"CompositeAlternatives__LoopBack (389)
LoopBack

dec=48"} + q390["CompositeAlternatives__LoopEnd (390)
LoopEnd
"] + q391{"CompositeAlternatives__Basic_3 (391)
Basic

dec=49"} + + q68 --> q385 + q385 -.->|"[CompositeGroup]"| q391 + q386 -->|"tok("|")"| q387 + q387 -.->|"[CompositeGroup]"| q388 + q388 --> q389 + q389 --> q386 + q389 --> q390 + q390 --> q69 + q391 --> q386 + q391 --> q390 ``` ## CompositeGroup ```mermaid flowchart TD - q48(["CompositeGroup__Start (48)
RuleStart"]) - q49(["CompositeGroup__Stop (49)
RuleStop"]) - q281["CompositeGroup__Basic_0 (281)
Basic
"] - q282["CompositeGroup__Basic_1 (282)
Basic
"] - q283["CompositeGroup__Basic_2 (283)
Basic
"] - q284{"CompositeGroup__LoopBack (284)
LoopBack

dec=34"} - q285["CompositeGroup__LoopEnd (285)
LoopEnd
"] - q286{"CompositeGroup__Basic_3 (286)
Basic

dec=35"} - - q48 --> q281 - q281 -.->|"[CompositeElement]"| q286 - q282 -.->|"[CompositeElement]"| q283 - q283 --> q284 - q284 --> q282 - q284 --> q285 - q285 --> q49 - q286 --> q282 - q286 --> q285 + q70(["CompositeGroup__Start (70)
RuleStart"]) + q71(["CompositeGroup__Stop (71)
RuleStop"]) + q392["CompositeGroup__Basic_0 (392)
Basic
"] + q393["CompositeGroup__Basic_1 (393)
Basic
"] + q394["CompositeGroup__Basic_2 (394)
Basic
"] + q395{"CompositeGroup__LoopBack (395)
LoopBack

dec=50"} + q396["CompositeGroup__LoopEnd (396)
LoopEnd
"] + q397{"CompositeGroup__Basic_3 (397)
Basic

dec=51"} + + q70 --> q392 + q392 -.->|"[CompositeElement]"| q397 + q393 -.->|"[CompositeElement]"| q394 + q394 --> q395 + q395 --> q393 + q395 --> q396 + q396 --> q71 + q397 --> q393 + q397 --> q396 ``` ## CompositeElement ```mermaid flowchart TD - q50(["CompositeElement__Start (50)
RuleStart"]) - q51(["CompositeElement__Stop (51)
RuleStop"]) - q287["CompositeElement__Basic_0 (287)
Basic
"] - q288["CompositeElement__Basic_1 (288)
Basic
"] - q289["CompositeElement__Basic_2 (289)
Basic
"] - q290["CompositeElement__Basic_3 (290)
Basic
"] - q291["CompositeElement_LeftParen (291)
Basic
"] - q292["CompositeElement__Basic_4 (292)
Basic
"] - q293["CompositeElement_RightParen (293)
Basic
"] - q294["CompositeElement__Basic_5 (294)
Basic
"] - q295{"CompositeElement__Basic_6 (295)
Basic

dec=36"} - q296["CompositeElement__BlockEnd_0 (296)
BlockEnd
"] - q297["CompositeElement_Cardinality_Asterisk (297)
Basic
"] - q298["CompositeElement__Basic_7 (298)
Basic
"] - q299["CompositeElement_Cardinality_Plus (299)
Basic
"] - q300["CompositeElement__Basic_8 (300)
Basic
"] - q301["CompositeElement_Cardinality_Question (301)
Basic
"] - q302["CompositeElement__Basic_9 (302)
Basic
"] - q303{"CompositeElement__Basic_10 (303)
Basic

dec=37"} - q304["CompositeElement__BlockEnd_1 (304)
BlockEnd
"] - q305{"CompositeElement__Basic_11 (305)
Basic

dec=38"} - - q50 --> q295 - q287 -.->|"[Keyword]"| q288 - q288 --> q296 - q289 -.->|"[RuleCall]"| q290 - q290 --> q296 - q291 -->|"tok("(")"| q292 - q292 -.->|"[CompositeAlternatives]"| q293 - q293 -->|"tok(")")"| q294 - q294 --> q296 - q295 --> q287 - q295 --> q289 - q295 --> q291 - q296 --> q305 - q297 -->|"tok("*")"| q298 - q298 --> q304 - q299 -->|"tok("+")"| q300 - q300 --> q304 - q301 -->|"tok("?")"| q302 - q302 --> q304 - q303 --> q297 - q303 --> q299 - q303 --> q301 - q304 --> q51 - q305 --> q303 - q305 --> q304 + q72(["CompositeElement__Start (72)
RuleStart"]) + q73(["CompositeElement__Stop (73)
RuleStop"]) + q398["CompositeElement__Basic_0 (398)
Basic
"] + q399["CompositeElement__Basic_1 (399)
Basic
"] + q400["CompositeElement__Basic_2 (400)
Basic
"] + q401["CompositeElement__Basic_3 (401)
Basic
"] + q402["CompositeElement_LeftParen (402)
Basic
"] + q403["CompositeElement__Basic_4 (403)
Basic
"] + q404["CompositeElement_RightParen (404)
Basic
"] + q405["CompositeElement__Basic_5 (405)
Basic
"] + q406{"CompositeElement__Basic_6 (406)
Basic

dec=52"} + q407["CompositeElement__BlockEnd (407)
BlockEnd
"] + q408["CompositeElement__Basic_7 (408)
Basic
"] + q409["CompositeElement__Basic_8 (409)
Basic
"] + q410{"CompositeElement__Basic_9 (410)
Basic

dec=53"} + + q72 --> q406 + q398 -.->|"[Keyword]"| q399 + q399 --> q407 + q400 -.->|"[RuleCall]"| q401 + q401 --> q407 + q402 -->|"tok("(")"| q403 + q403 -.->|"[CompositeAlternatives]"| q404 + q404 -->|"tok(")")"| q405 + q405 --> q407 + q406 --> q398 + q406 --> q400 + q406 --> q402 + q407 --> q410 + q408 -->|"tok(Cardinality)"| q409 + q409 --> q73 + q410 --> q408 + q410 --> q409 ``` diff --git a/internal/grammar/atn_gen.go b/internal/grammar/atn_gen.go index 0d5bdc1b..8b3512b7 100644 --- a/internal/grammar/atn_gen.go +++ b/internal/grammar/atn_gen.go @@ -26,10 +26,32 @@ const ( PrimitiveType__Stop ParserRule__Start ParserRule__Stop - Token__Start - Token__Stop + TokenDecl__Start + TokenDecl__Stop + TokenContent__Start + TokenContent__Stop + RegexpTokenContent__Start + RegexpTokenContent__Stop + KeywordTokenContent__Start + KeywordTokenContent__Stop + TokenCommand__Start + TokenCommand__Stop TokenGroup__Start TokenGroup__Stop + TokenMode__Start + TokenMode__Stop + TokenModeMember__Start + TokenModeMember__Stop + TokenDeclUsage__Start + TokenDeclUsage__Stop + TokenGroupUsage__Start + TokenGroupUsage__Stop + TokenUsage__Start + TokenUsage__Stop + KeywordUsage__Start + KeywordUsage__Stop + KeywordSelector__Start + KeywordSelector__Stop Alternatives__Start Alternatives__Stop Group__Start @@ -76,6 +98,8 @@ const ( Grammar__Basic_10 Grammar__Basic_11 Grammar__Basic_12 + Grammar__Basic_13 + Grammar__Basic_14 Grammar__BlockEnd Grammar__LoopEntry Grammar__LoopEnd @@ -142,38 +166,131 @@ const ( ParserRule_Semicolon ParserRule__Basic_5 ParserRule__Basic_6 - Token_Type_hidden - Token__Basic_0 - Token_Type_comment - Token__Basic_1 - Token__Basic_2 - Token__BlockEnd - Token__Basic_3 - Token_token - Token_Name_ID - Token_Colon - Token_Regexp_RegexLiteral - Token_Semicolon - Token__Basic_4 - Token__Basic_5 + TokenDecl__Basic_0 + TokenDecl__Basic_1 + TokenDecl__Basic_2 + TokenDecl_token + TokenDecl_Name_ID + TokenDecl_Colon + TokenDecl__Basic_3 + TokenDecl__Basic_4 + TokenDecl__Basic_5 + TokenDecl__Basic_6 + TokenDecl_Semicolon + TokenDecl__Basic_7 + TokenDecl__Basic_8 + TokenContent__Basic_0 + TokenContent__Basic_1 + TokenContent__Basic_2 + TokenContent__Basic_3 + TokenContent__Basic_4 + TokenContent__BlockEnd + RegexpTokenContent_Regexp_RegexLiteral + RegexpTokenContent__Basic + KeywordTokenContent__Basic_0 + KeywordTokenContent__Basic_1 + TokenCommand_DashGreaterThan + TokenCommand_Type_push + TokenCommand__Basic_0 + TokenCommand_Type_pop + TokenCommand__Basic_1 + TokenCommand_Type_mode + TokenCommand__Basic_2 + TokenCommand__Basic_3 + TokenCommand__BlockEnd_0 + TokenCommand_LeftParen + TokenCommand_Mode_ID + TokenCommand__Basic_4 + TokenCommand_Default_default + TokenCommand__Basic_5 + TokenCommand__Basic_6 + TokenCommand__BlockEnd_1 + TokenCommand_RightParen + TokenCommand__Basic_7 + TokenCommand__Basic_8 + TokenGroup__Basic_0 + TokenGroup__Basic_1 + TokenGroup__Basic_2 TokenGroup_token TokenGroup_group TokenGroup_Name_ID TokenGroup_LeftBrace TokenGroup_TokenRefs_ID - TokenGroup__Basic_0 - TokenGroup_keywords - TokenGroup_Regexps_RegexLiteral - TokenGroup__Basic_1 - TokenGroup__Basic_2 TokenGroup__Basic_3 TokenGroup__Basic_4 + TokenGroup__Basic_5 + TokenGroup_keywords + TokenGroup_KeywordSelectors_RegexLiteral + TokenGroup__Basic_6 + TokenGroup__Basic_7 TokenGroup__BlockEnd TokenGroup__LoopEntry TokenGroup__LoopEnd TokenGroup__LoopBack TokenGroup_RightBrace - TokenGroup__Basic_5 + TokenGroup__Basic_8 + TokenGroup__Basic_9 + TokenGroup__Basic_10 + TokenGroup_Semicolon + TokenGroup__Basic_11 + TokenGroup__Basic_12 + TokenMode_token + TokenMode_mode + TokenMode_Name_ID + TokenMode__Basic_0 + TokenMode_Default_default + TokenMode__Basic_1 + TokenMode__Basic_2 + TokenMode__BlockEnd + TokenMode_LeftBrace + TokenMode__Basic_3 + TokenMode__Basic_4 + TokenMode__LoopEntry + TokenMode__LoopEnd + TokenMode__LoopBack + TokenMode_RightBrace + TokenMode__Basic_5 + TokenModeMember__Basic_0 + TokenModeMember__Basic_1 + TokenModeMember__Basic_2 + TokenModeMember__Basic_3 + TokenModeMember__Basic_4 + TokenModeMember__Basic_5 + TokenModeMember__Basic_6 + TokenModeMember__Basic_7 + TokenModeMember__Basic_8 + TokenModeMember__Basic_9 + TokenModeMember__Basic_10 + TokenModeMember__BlockEnd + TokenDeclUsage__Basic_0 + TokenDeclUsage__Basic_1 + TokenGroupUsage__Basic_0 + TokenGroupUsage__Basic_1 + TokenUsage__Basic_0 + TokenUsage__Basic_1 + TokenUsage__Basic_2 + TokenUsage_TokenRef_ID + TokenUsage__Basic_3 + TokenUsage__Basic_4 + TokenUsage__Basic_5 + TokenUsage_Semicolon + TokenUsage__Basic_6 + TokenUsage__Basic_7 + KeywordUsage__Basic_0 + KeywordUsage__Basic_1 + KeywordUsage__Basic_2 + KeywordUsage__Basic_3 + KeywordUsage__Basic_4 + KeywordUsage__Basic_5 + KeywordUsage__Basic_6 + KeywordUsage_Semicolon + KeywordUsage__Basic_7 + KeywordUsage__Basic_8 + KeywordSelector_keywords + KeywordSelector_Selector_RegexLiteral + KeywordSelector_Semicolon + KeywordSelector__Basic_0 + KeywordSelector__Basic_1 Alternatives__Basic_0 Alternatives_Pipe Alternatives__Basic_1 @@ -200,16 +317,10 @@ const ( Element_RightParen Element__Basic_9 Element__Basic_10 - Element__BlockEnd_0 - Element_Cardinality_Asterisk + Element__BlockEnd Element__Basic_11 - Element_Cardinality_Plus Element__Basic_12 - Element_Cardinality_Question Element__Basic_13 - Element__Basic_14 - Element__BlockEnd_1 - Element__Basic_15 Keyword_Value_StringLiteral Keyword__Basic Assignment_Property_ID @@ -304,16 +415,10 @@ const ( CompositeElement_RightParen CompositeElement__Basic_5 CompositeElement__Basic_6 - CompositeElement__BlockEnd_0 - CompositeElement_Cardinality_Asterisk + CompositeElement__BlockEnd CompositeElement__Basic_7 - CompositeElement_Cardinality_Plus CompositeElement__Basic_8 - CompositeElement_Cardinality_Question CompositeElement__Basic_9 - CompositeElement__Basic_10 - CompositeElement__BlockEnd_1 - CompositeElement__Basic_11 ) var once sync.Once @@ -326,7 +431,7 @@ func ATN() *parser.RuntimeATN { return atn } func BuildATN() *parser.RuntimeATN { - states := make([]*parser.RuntimeATNState, 306) + states := make([]*parser.RuntimeATNState, 411) states[Grammar__Start] = parser.NewATNState(Grammar__Start, parser.ATNRuleStart, true) states[Grammar__Stop] = parser.NewATNState(Grammar__Stop, parser.ATNRuleStop, false) states[Interface__Start] = parser.NewATNState(Interface__Start, parser.ATNRuleStart, true) @@ -345,10 +450,32 @@ func BuildATN() *parser.RuntimeATN { states[PrimitiveType__Stop] = parser.NewATNState(PrimitiveType__Stop, parser.ATNRuleStop, false) states[ParserRule__Start] = parser.NewATNState(ParserRule__Start, parser.ATNRuleStart, true) states[ParserRule__Stop] = parser.NewATNState(ParserRule__Stop, parser.ATNRuleStop, false) - states[Token__Start] = parser.NewATNState(Token__Start, parser.ATNRuleStart, true) - states[Token__Stop] = parser.NewATNState(Token__Stop, parser.ATNRuleStop, false) + states[TokenDecl__Start] = parser.NewATNState(TokenDecl__Start, parser.ATNRuleStart, true) + states[TokenDecl__Stop] = parser.NewATNState(TokenDecl__Stop, parser.ATNRuleStop, false) + states[TokenContent__Start] = parser.NewATNState(TokenContent__Start, parser.ATNRuleStart, true) + states[TokenContent__Stop] = parser.NewATNState(TokenContent__Stop, parser.ATNRuleStop, false) + states[RegexpTokenContent__Start] = parser.NewATNState(RegexpTokenContent__Start, parser.ATNRuleStart, true) + states[RegexpTokenContent__Stop] = parser.NewATNState(RegexpTokenContent__Stop, parser.ATNRuleStop, false) + states[KeywordTokenContent__Start] = parser.NewATNState(KeywordTokenContent__Start, parser.ATNRuleStart, true) + states[KeywordTokenContent__Stop] = parser.NewATNState(KeywordTokenContent__Stop, parser.ATNRuleStop, false) + states[TokenCommand__Start] = parser.NewATNState(TokenCommand__Start, parser.ATNRuleStart, true) + states[TokenCommand__Stop] = parser.NewATNState(TokenCommand__Stop, parser.ATNRuleStop, false) states[TokenGroup__Start] = parser.NewATNState(TokenGroup__Start, parser.ATNRuleStart, true) states[TokenGroup__Stop] = parser.NewATNState(TokenGroup__Stop, parser.ATNRuleStop, false) + states[TokenMode__Start] = parser.NewATNState(TokenMode__Start, parser.ATNRuleStart, true) + states[TokenMode__Stop] = parser.NewATNState(TokenMode__Stop, parser.ATNRuleStop, false) + states[TokenModeMember__Start] = parser.NewATNState(TokenModeMember__Start, parser.ATNRuleStart, true) + states[TokenModeMember__Stop] = parser.NewATNState(TokenModeMember__Stop, parser.ATNRuleStop, false) + states[TokenDeclUsage__Start] = parser.NewATNState(TokenDeclUsage__Start, parser.ATNRuleStart, true) + states[TokenDeclUsage__Stop] = parser.NewATNState(TokenDeclUsage__Stop, parser.ATNRuleStop, false) + states[TokenGroupUsage__Start] = parser.NewATNState(TokenGroupUsage__Start, parser.ATNRuleStart, true) + states[TokenGroupUsage__Stop] = parser.NewATNState(TokenGroupUsage__Stop, parser.ATNRuleStop, false) + states[TokenUsage__Start] = parser.NewATNState(TokenUsage__Start, parser.ATNRuleStart, true) + states[TokenUsage__Stop] = parser.NewATNState(TokenUsage__Stop, parser.ATNRuleStop, false) + states[KeywordUsage__Start] = parser.NewATNState(KeywordUsage__Start, parser.ATNRuleStart, true) + states[KeywordUsage__Stop] = parser.NewATNState(KeywordUsage__Stop, parser.ATNRuleStop, false) + states[KeywordSelector__Start] = parser.NewATNState(KeywordSelector__Start, parser.ATNRuleStart, true) + states[KeywordSelector__Stop] = parser.NewATNState(KeywordSelector__Stop, parser.ATNRuleStop, false) states[Alternatives__Start] = parser.NewATNState(Alternatives__Start, parser.ATNRuleStart, true) states[Alternatives__Stop] = parser.NewATNState(Alternatives__Stop, parser.ATNRuleStop, false) states[Group__Start] = parser.NewATNState(Group__Start, parser.ATNRuleStart, true) @@ -394,7 +521,9 @@ func BuildATN() *parser.RuntimeATN { states[Grammar__Basic_9] = parser.NewATNState(Grammar__Basic_9, parser.ATNBasic, true) states[Grammar__Basic_10] = parser.NewATNState(Grammar__Basic_10, parser.ATNBasic, true) states[Grammar__Basic_11] = parser.NewATNState(Grammar__Basic_11, parser.ATNBasic, true) - states[Grammar__Basic_12] = parser.NewATNState(Grammar__Basic_12, parser.ATNBasic, true).SetDecision(1) + states[Grammar__Basic_12] = parser.NewATNState(Grammar__Basic_12, parser.ATNBasic, true) + states[Grammar__Basic_13] = parser.NewATNState(Grammar__Basic_13, parser.ATNBasic, true) + states[Grammar__Basic_14] = parser.NewATNState(Grammar__Basic_14, parser.ATNBasic, true).SetDecision(1) states[Grammar__BlockEnd] = parser.NewATNState(Grammar__BlockEnd, parser.ATNBlockEnd, true) states[Grammar__LoopEntry] = parser.NewATNState(Grammar__LoopEntry, parser.ATNLoopEntry, true).SetDecision(2) states[Grammar__LoopEnd] = parser.NewATNState(Grammar__LoopEnd, parser.ATNLoopEnd, true) @@ -461,51 +590,144 @@ func BuildATN() *parser.RuntimeATN { states[ParserRule_Semicolon] = parser.NewATNState(ParserRule_Semicolon, parser.ATNBasic, false) states[ParserRule__Basic_5] = parser.NewATNState(ParserRule__Basic_5, parser.ATNBasic, true) states[ParserRule__Basic_6] = parser.NewATNState(ParserRule__Basic_6, parser.ATNBasic, true).SetDecision(10) - states[Token_Type_hidden] = parser.NewATNState(Token_Type_hidden, parser.ATNBasic, false) - states[Token__Basic_0] = parser.NewATNState(Token__Basic_0, parser.ATNBasic, true) - states[Token_Type_comment] = parser.NewATNState(Token_Type_comment, parser.ATNBasic, false) - states[Token__Basic_1] = parser.NewATNState(Token__Basic_1, parser.ATNBasic, true) - states[Token__Basic_2] = parser.NewATNState(Token__Basic_2, parser.ATNBasic, true).SetDecision(11) - states[Token__BlockEnd] = parser.NewATNState(Token__BlockEnd, parser.ATNBlockEnd, true) - states[Token__Basic_3] = parser.NewATNState(Token__Basic_3, parser.ATNBasic, true).SetDecision(12) - states[Token_token] = parser.NewATNState(Token_token, parser.ATNBasic, false) - states[Token_Name_ID] = parser.NewATNState(Token_Name_ID, parser.ATNBasic, false) - states[Token_Colon] = parser.NewATNState(Token_Colon, parser.ATNBasic, false) - states[Token_Regexp_RegexLiteral] = parser.NewATNState(Token_Regexp_RegexLiteral, parser.ATNBasic, false) - states[Token_Semicolon] = parser.NewATNState(Token_Semicolon, parser.ATNBasic, false) - states[Token__Basic_4] = parser.NewATNState(Token__Basic_4, parser.ATNBasic, true) - states[Token__Basic_5] = parser.NewATNState(Token__Basic_5, parser.ATNBasic, true).SetDecision(13) + states[TokenDecl__Basic_0] = parser.NewATNState(TokenDecl__Basic_0, parser.ATNBasic, false) + states[TokenDecl__Basic_1] = parser.NewATNState(TokenDecl__Basic_1, parser.ATNBasic, true) + states[TokenDecl__Basic_2] = parser.NewATNState(TokenDecl__Basic_2, parser.ATNBasic, true).SetDecision(11) + states[TokenDecl_token] = parser.NewATNState(TokenDecl_token, parser.ATNBasic, false) + states[TokenDecl_Name_ID] = parser.NewATNState(TokenDecl_Name_ID, parser.ATNBasic, false) + states[TokenDecl_Colon] = parser.NewATNState(TokenDecl_Colon, parser.ATNBasic, false) + states[TokenDecl__Basic_3] = parser.NewATNState(TokenDecl__Basic_3, parser.ATNBasic, true) + states[TokenDecl__Basic_4] = parser.NewATNState(TokenDecl__Basic_4, parser.ATNBasic, true) + states[TokenDecl__Basic_5] = parser.NewATNState(TokenDecl__Basic_5, parser.ATNBasic, true) + states[TokenDecl__Basic_6] = parser.NewATNState(TokenDecl__Basic_6, parser.ATNBasic, true).SetDecision(12) + states[TokenDecl_Semicolon] = parser.NewATNState(TokenDecl_Semicolon, parser.ATNBasic, false) + states[TokenDecl__Basic_7] = parser.NewATNState(TokenDecl__Basic_7, parser.ATNBasic, true) + states[TokenDecl__Basic_8] = parser.NewATNState(TokenDecl__Basic_8, parser.ATNBasic, true).SetDecision(13) + states[TokenContent__Basic_0] = parser.NewATNState(TokenContent__Basic_0, parser.ATNBasic, true) + states[TokenContent__Basic_1] = parser.NewATNState(TokenContent__Basic_1, parser.ATNBasic, true) + states[TokenContent__Basic_2] = parser.NewATNState(TokenContent__Basic_2, parser.ATNBasic, true) + states[TokenContent__Basic_3] = parser.NewATNState(TokenContent__Basic_3, parser.ATNBasic, true) + states[TokenContent__Basic_4] = parser.NewATNState(TokenContent__Basic_4, parser.ATNBasic, true).SetDecision(14) + states[TokenContent__BlockEnd] = parser.NewATNState(TokenContent__BlockEnd, parser.ATNBlockEnd, true) + states[RegexpTokenContent_Regexp_RegexLiteral] = parser.NewATNState(RegexpTokenContent_Regexp_RegexLiteral, parser.ATNBasic, false) + states[RegexpTokenContent__Basic] = parser.NewATNState(RegexpTokenContent__Basic, parser.ATNBasic, true) + states[KeywordTokenContent__Basic_0] = parser.NewATNState(KeywordTokenContent__Basic_0, parser.ATNBasic, true) + states[KeywordTokenContent__Basic_1] = parser.NewATNState(KeywordTokenContent__Basic_1, parser.ATNBasic, true) + states[TokenCommand_DashGreaterThan] = parser.NewATNState(TokenCommand_DashGreaterThan, parser.ATNBasic, false) + states[TokenCommand_Type_push] = parser.NewATNState(TokenCommand_Type_push, parser.ATNBasic, false) + states[TokenCommand__Basic_0] = parser.NewATNState(TokenCommand__Basic_0, parser.ATNBasic, true) + states[TokenCommand_Type_pop] = parser.NewATNState(TokenCommand_Type_pop, parser.ATNBasic, false) + states[TokenCommand__Basic_1] = parser.NewATNState(TokenCommand__Basic_1, parser.ATNBasic, true) + states[TokenCommand_Type_mode] = parser.NewATNState(TokenCommand_Type_mode, parser.ATNBasic, false) + states[TokenCommand__Basic_2] = parser.NewATNState(TokenCommand__Basic_2, parser.ATNBasic, true) + states[TokenCommand__Basic_3] = parser.NewATNState(TokenCommand__Basic_3, parser.ATNBasic, true).SetDecision(15) + states[TokenCommand__BlockEnd_0] = parser.NewATNState(TokenCommand__BlockEnd_0, parser.ATNBlockEnd, true) + states[TokenCommand_LeftParen] = parser.NewATNState(TokenCommand_LeftParen, parser.ATNBasic, false) + states[TokenCommand_Mode_ID] = parser.NewATNState(TokenCommand_Mode_ID, parser.ATNBasic, false) + states[TokenCommand__Basic_4] = parser.NewATNState(TokenCommand__Basic_4, parser.ATNBasic, true) + states[TokenCommand_Default_default] = parser.NewATNState(TokenCommand_Default_default, parser.ATNBasic, false) + states[TokenCommand__Basic_5] = parser.NewATNState(TokenCommand__Basic_5, parser.ATNBasic, true) + states[TokenCommand__Basic_6] = parser.NewATNState(TokenCommand__Basic_6, parser.ATNBasic, true).SetDecision(16) + states[TokenCommand__BlockEnd_1] = parser.NewATNState(TokenCommand__BlockEnd_1, parser.ATNBlockEnd, true) + states[TokenCommand_RightParen] = parser.NewATNState(TokenCommand_RightParen, parser.ATNBasic, false) + states[TokenCommand__Basic_7] = parser.NewATNState(TokenCommand__Basic_7, parser.ATNBasic, true) + states[TokenCommand__Basic_8] = parser.NewATNState(TokenCommand__Basic_8, parser.ATNBasic, true).SetDecision(17) + states[TokenGroup__Basic_0] = parser.NewATNState(TokenGroup__Basic_0, parser.ATNBasic, false) + states[TokenGroup__Basic_1] = parser.NewATNState(TokenGroup__Basic_1, parser.ATNBasic, true) + states[TokenGroup__Basic_2] = parser.NewATNState(TokenGroup__Basic_2, parser.ATNBasic, true).SetDecision(18) states[TokenGroup_token] = parser.NewATNState(TokenGroup_token, parser.ATNBasic, false) states[TokenGroup_group] = parser.NewATNState(TokenGroup_group, parser.ATNBasic, false) states[TokenGroup_Name_ID] = parser.NewATNState(TokenGroup_Name_ID, parser.ATNBasic, false) states[TokenGroup_LeftBrace] = parser.NewATNState(TokenGroup_LeftBrace, parser.ATNBasic, false) states[TokenGroup_TokenRefs_ID] = parser.NewATNState(TokenGroup_TokenRefs_ID, parser.ATNBasic, false) - states[TokenGroup__Basic_0] = parser.NewATNState(TokenGroup__Basic_0, parser.ATNBasic, true) - states[TokenGroup_keywords] = parser.NewATNState(TokenGroup_keywords, parser.ATNBasic, false) - states[TokenGroup_Regexps_RegexLiteral] = parser.NewATNState(TokenGroup_Regexps_RegexLiteral, parser.ATNBasic, false) - states[TokenGroup__Basic_1] = parser.NewATNState(TokenGroup__Basic_1, parser.ATNBasic, true) - states[TokenGroup__Basic_2] = parser.NewATNState(TokenGroup__Basic_2, parser.ATNBasic, true) states[TokenGroup__Basic_3] = parser.NewATNState(TokenGroup__Basic_3, parser.ATNBasic, true) - states[TokenGroup__Basic_4] = parser.NewATNState(TokenGroup__Basic_4, parser.ATNBasic, true).SetDecision(14) + states[TokenGroup__Basic_4] = parser.NewATNState(TokenGroup__Basic_4, parser.ATNBasic, true) + states[TokenGroup__Basic_5] = parser.NewATNState(TokenGroup__Basic_5, parser.ATNBasic, true) + states[TokenGroup_keywords] = parser.NewATNState(TokenGroup_keywords, parser.ATNBasic, false) + states[TokenGroup_KeywordSelectors_RegexLiteral] = parser.NewATNState(TokenGroup_KeywordSelectors_RegexLiteral, parser.ATNBasic, false) + states[TokenGroup__Basic_6] = parser.NewATNState(TokenGroup__Basic_6, parser.ATNBasic, true) + states[TokenGroup__Basic_7] = parser.NewATNState(TokenGroup__Basic_7, parser.ATNBasic, true).SetDecision(19) states[TokenGroup__BlockEnd] = parser.NewATNState(TokenGroup__BlockEnd, parser.ATNBlockEnd, true) - states[TokenGroup__LoopEntry] = parser.NewATNState(TokenGroup__LoopEntry, parser.ATNLoopEntry, true).SetDecision(15) + states[TokenGroup__LoopEntry] = parser.NewATNState(TokenGroup__LoopEntry, parser.ATNLoopEntry, true).SetDecision(20) states[TokenGroup__LoopEnd] = parser.NewATNState(TokenGroup__LoopEnd, parser.ATNLoopEnd, true) states[TokenGroup__LoopBack] = parser.NewATNState(TokenGroup__LoopBack, parser.ATNLoopBack, true) states[TokenGroup_RightBrace] = parser.NewATNState(TokenGroup_RightBrace, parser.ATNBasic, false) - states[TokenGroup__Basic_5] = parser.NewATNState(TokenGroup__Basic_5, parser.ATNBasic, true) + states[TokenGroup__Basic_8] = parser.NewATNState(TokenGroup__Basic_8, parser.ATNBasic, true) + states[TokenGroup__Basic_9] = parser.NewATNState(TokenGroup__Basic_9, parser.ATNBasic, true) + states[TokenGroup__Basic_10] = parser.NewATNState(TokenGroup__Basic_10, parser.ATNBasic, true).SetDecision(21) + states[TokenGroup_Semicolon] = parser.NewATNState(TokenGroup_Semicolon, parser.ATNBasic, false) + states[TokenGroup__Basic_11] = parser.NewATNState(TokenGroup__Basic_11, parser.ATNBasic, true) + states[TokenGroup__Basic_12] = parser.NewATNState(TokenGroup__Basic_12, parser.ATNBasic, true).SetDecision(22) + states[TokenMode_token] = parser.NewATNState(TokenMode_token, parser.ATNBasic, false) + states[TokenMode_mode] = parser.NewATNState(TokenMode_mode, parser.ATNBasic, false) + states[TokenMode_Name_ID] = parser.NewATNState(TokenMode_Name_ID, parser.ATNBasic, false) + states[TokenMode__Basic_0] = parser.NewATNState(TokenMode__Basic_0, parser.ATNBasic, true) + states[TokenMode_Default_default] = parser.NewATNState(TokenMode_Default_default, parser.ATNBasic, false) + states[TokenMode__Basic_1] = parser.NewATNState(TokenMode__Basic_1, parser.ATNBasic, true) + states[TokenMode__Basic_2] = parser.NewATNState(TokenMode__Basic_2, parser.ATNBasic, true).SetDecision(23) + states[TokenMode__BlockEnd] = parser.NewATNState(TokenMode__BlockEnd, parser.ATNBlockEnd, true) + states[TokenMode_LeftBrace] = parser.NewATNState(TokenMode_LeftBrace, parser.ATNBasic, false) + states[TokenMode__Basic_3] = parser.NewATNState(TokenMode__Basic_3, parser.ATNBasic, true) + states[TokenMode__Basic_4] = parser.NewATNState(TokenMode__Basic_4, parser.ATNBasic, true) + states[TokenMode__LoopEntry] = parser.NewATNState(TokenMode__LoopEntry, parser.ATNLoopEntry, true).SetDecision(24) + states[TokenMode__LoopEnd] = parser.NewATNState(TokenMode__LoopEnd, parser.ATNLoopEnd, true) + states[TokenMode__LoopBack] = parser.NewATNState(TokenMode__LoopBack, parser.ATNLoopBack, true) + states[TokenMode_RightBrace] = parser.NewATNState(TokenMode_RightBrace, parser.ATNBasic, false) + states[TokenMode__Basic_5] = parser.NewATNState(TokenMode__Basic_5, parser.ATNBasic, true) + states[TokenModeMember__Basic_0] = parser.NewATNState(TokenModeMember__Basic_0, parser.ATNBasic, true) + states[TokenModeMember__Basic_1] = parser.NewATNState(TokenModeMember__Basic_1, parser.ATNBasic, true) + states[TokenModeMember__Basic_2] = parser.NewATNState(TokenModeMember__Basic_2, parser.ATNBasic, true) + states[TokenModeMember__Basic_3] = parser.NewATNState(TokenModeMember__Basic_3, parser.ATNBasic, true) + states[TokenModeMember__Basic_4] = parser.NewATNState(TokenModeMember__Basic_4, parser.ATNBasic, true) + states[TokenModeMember__Basic_5] = parser.NewATNState(TokenModeMember__Basic_5, parser.ATNBasic, true) + states[TokenModeMember__Basic_6] = parser.NewATNState(TokenModeMember__Basic_6, parser.ATNBasic, true) + states[TokenModeMember__Basic_7] = parser.NewATNState(TokenModeMember__Basic_7, parser.ATNBasic, true) + states[TokenModeMember__Basic_8] = parser.NewATNState(TokenModeMember__Basic_8, parser.ATNBasic, true) + states[TokenModeMember__Basic_9] = parser.NewATNState(TokenModeMember__Basic_9, parser.ATNBasic, true) + states[TokenModeMember__Basic_10] = parser.NewATNState(TokenModeMember__Basic_10, parser.ATNBasic, true).SetDecision(25) + states[TokenModeMember__BlockEnd] = parser.NewATNState(TokenModeMember__BlockEnd, parser.ATNBlockEnd, true) + states[TokenDeclUsage__Basic_0] = parser.NewATNState(TokenDeclUsage__Basic_0, parser.ATNBasic, true) + states[TokenDeclUsage__Basic_1] = parser.NewATNState(TokenDeclUsage__Basic_1, parser.ATNBasic, true) + states[TokenGroupUsage__Basic_0] = parser.NewATNState(TokenGroupUsage__Basic_0, parser.ATNBasic, true) + states[TokenGroupUsage__Basic_1] = parser.NewATNState(TokenGroupUsage__Basic_1, parser.ATNBasic, true) + states[TokenUsage__Basic_0] = parser.NewATNState(TokenUsage__Basic_0, parser.ATNBasic, false) + states[TokenUsage__Basic_1] = parser.NewATNState(TokenUsage__Basic_1, parser.ATNBasic, true) + states[TokenUsage__Basic_2] = parser.NewATNState(TokenUsage__Basic_2, parser.ATNBasic, true).SetDecision(26) + states[TokenUsage_TokenRef_ID] = parser.NewATNState(TokenUsage_TokenRef_ID, parser.ATNBasic, false) + states[TokenUsage__Basic_3] = parser.NewATNState(TokenUsage__Basic_3, parser.ATNBasic, true) + states[TokenUsage__Basic_4] = parser.NewATNState(TokenUsage__Basic_4, parser.ATNBasic, true) + states[TokenUsage__Basic_5] = parser.NewATNState(TokenUsage__Basic_5, parser.ATNBasic, true).SetDecision(27) + states[TokenUsage_Semicolon] = parser.NewATNState(TokenUsage_Semicolon, parser.ATNBasic, false) + states[TokenUsage__Basic_6] = parser.NewATNState(TokenUsage__Basic_6, parser.ATNBasic, true) + states[TokenUsage__Basic_7] = parser.NewATNState(TokenUsage__Basic_7, parser.ATNBasic, true).SetDecision(28) + states[KeywordUsage__Basic_0] = parser.NewATNState(KeywordUsage__Basic_0, parser.ATNBasic, false) + states[KeywordUsage__Basic_1] = parser.NewATNState(KeywordUsage__Basic_1, parser.ATNBasic, true) + states[KeywordUsage__Basic_2] = parser.NewATNState(KeywordUsage__Basic_2, parser.ATNBasic, true).SetDecision(29) + states[KeywordUsage__Basic_3] = parser.NewATNState(KeywordUsage__Basic_3, parser.ATNBasic, true) + states[KeywordUsage__Basic_4] = parser.NewATNState(KeywordUsage__Basic_4, parser.ATNBasic, true) + states[KeywordUsage__Basic_5] = parser.NewATNState(KeywordUsage__Basic_5, parser.ATNBasic, true) + states[KeywordUsage__Basic_6] = parser.NewATNState(KeywordUsage__Basic_6, parser.ATNBasic, true).SetDecision(30) + states[KeywordUsage_Semicolon] = parser.NewATNState(KeywordUsage_Semicolon, parser.ATNBasic, false) + states[KeywordUsage__Basic_7] = parser.NewATNState(KeywordUsage__Basic_7, parser.ATNBasic, true) + states[KeywordUsage__Basic_8] = parser.NewATNState(KeywordUsage__Basic_8, parser.ATNBasic, true).SetDecision(31) + states[KeywordSelector_keywords] = parser.NewATNState(KeywordSelector_keywords, parser.ATNBasic, false) + states[KeywordSelector_Selector_RegexLiteral] = parser.NewATNState(KeywordSelector_Selector_RegexLiteral, parser.ATNBasic, false) + states[KeywordSelector_Semicolon] = parser.NewATNState(KeywordSelector_Semicolon, parser.ATNBasic, false) + states[KeywordSelector__Basic_0] = parser.NewATNState(KeywordSelector__Basic_0, parser.ATNBasic, true) + states[KeywordSelector__Basic_1] = parser.NewATNState(KeywordSelector__Basic_1, parser.ATNBasic, true).SetDecision(32) states[Alternatives__Basic_0] = parser.NewATNState(Alternatives__Basic_0, parser.ATNBasic, true) states[Alternatives_Pipe] = parser.NewATNState(Alternatives_Pipe, parser.ATNBasic, false) states[Alternatives__Basic_1] = parser.NewATNState(Alternatives__Basic_1, parser.ATNBasic, true) states[Alternatives__Basic_2] = parser.NewATNState(Alternatives__Basic_2, parser.ATNBasic, true) - states[Alternatives__LoopBack] = parser.NewATNState(Alternatives__LoopBack, parser.ATNLoopBack, true).SetDecision(16) + states[Alternatives__LoopBack] = parser.NewATNState(Alternatives__LoopBack, parser.ATNLoopBack, true).SetDecision(33) states[Alternatives__LoopEnd] = parser.NewATNState(Alternatives__LoopEnd, parser.ATNLoopEnd, true) - states[Alternatives__Basic_3] = parser.NewATNState(Alternatives__Basic_3, parser.ATNBasic, true).SetDecision(17) + states[Alternatives__Basic_3] = parser.NewATNState(Alternatives__Basic_3, parser.ATNBasic, true).SetDecision(34) states[Group__Basic_0] = parser.NewATNState(Group__Basic_0, parser.ATNBasic, true) states[Group__Basic_1] = parser.NewATNState(Group__Basic_1, parser.ATNBasic, true) states[Group__Basic_2] = parser.NewATNState(Group__Basic_2, parser.ATNBasic, true) - states[Group__LoopBack] = parser.NewATNState(Group__LoopBack, parser.ATNLoopBack, true).SetDecision(18) + states[Group__LoopBack] = parser.NewATNState(Group__LoopBack, parser.ATNLoopBack, true).SetDecision(35) states[Group__LoopEnd] = parser.NewATNState(Group__LoopEnd, parser.ATNLoopEnd, true) - states[Group__Basic_3] = parser.NewATNState(Group__Basic_3, parser.ATNBasic, true).SetDecision(19) + states[Group__Basic_3] = parser.NewATNState(Group__Basic_3, parser.ATNBasic, true).SetDecision(36) states[Element__Basic_0] = parser.NewATNState(Element__Basic_0, parser.ATNBasic, true) states[Element__Basic_1] = parser.NewATNState(Element__Basic_1, parser.ATNBasic, true) states[Element__Basic_2] = parser.NewATNState(Element__Basic_2, parser.ATNBasic, true) @@ -518,17 +740,11 @@ func BuildATN() *parser.RuntimeATN { states[Element__Basic_8] = parser.NewATNState(Element__Basic_8, parser.ATNBasic, true) states[Element_RightParen] = parser.NewATNState(Element_RightParen, parser.ATNBasic, false) states[Element__Basic_9] = parser.NewATNState(Element__Basic_9, parser.ATNBasic, true) - states[Element__Basic_10] = parser.NewATNState(Element__Basic_10, parser.ATNBasic, true).SetDecision(20) - states[Element__BlockEnd_0] = parser.NewATNState(Element__BlockEnd_0, parser.ATNBlockEnd, true) - states[Element_Cardinality_Asterisk] = parser.NewATNState(Element_Cardinality_Asterisk, parser.ATNBasic, false) - states[Element__Basic_11] = parser.NewATNState(Element__Basic_11, parser.ATNBasic, true) - states[Element_Cardinality_Plus] = parser.NewATNState(Element_Cardinality_Plus, parser.ATNBasic, false) + states[Element__Basic_10] = parser.NewATNState(Element__Basic_10, parser.ATNBasic, true).SetDecision(37) + states[Element__BlockEnd] = parser.NewATNState(Element__BlockEnd, parser.ATNBlockEnd, true) + states[Element__Basic_11] = parser.NewATNState(Element__Basic_11, parser.ATNBasic, false) states[Element__Basic_12] = parser.NewATNState(Element__Basic_12, parser.ATNBasic, true) - states[Element_Cardinality_Question] = parser.NewATNState(Element_Cardinality_Question, parser.ATNBasic, false) - states[Element__Basic_13] = parser.NewATNState(Element__Basic_13, parser.ATNBasic, true) - states[Element__Basic_14] = parser.NewATNState(Element__Basic_14, parser.ATNBasic, true).SetDecision(21) - states[Element__BlockEnd_1] = parser.NewATNState(Element__BlockEnd_1, parser.ATNBlockEnd, true) - states[Element__Basic_15] = parser.NewATNState(Element__Basic_15, parser.ATNBasic, true).SetDecision(22) + states[Element__Basic_13] = parser.NewATNState(Element__Basic_13, parser.ATNBasic, true).SetDecision(38) states[Keyword_Value_StringLiteral] = parser.NewATNState(Keyword_Value_StringLiteral, parser.ATNBasic, false) states[Keyword__Basic] = parser.NewATNState(Keyword__Basic, parser.ATNBasic, true) states[Assignment_Property_ID] = parser.NewATNState(Assignment_Property_ID, parser.ATNBasic, false) @@ -538,7 +754,7 @@ func BuildATN() *parser.RuntimeATN { states[Assignment__Basic_1] = parser.NewATNState(Assignment__Basic_1, parser.ATNBasic, true) states[Assignment_Operator_QuestionEquals] = parser.NewATNState(Assignment_Operator_QuestionEquals, parser.ATNBasic, false) states[Assignment__Basic_2] = parser.NewATNState(Assignment__Basic_2, parser.ATNBasic, true) - states[Assignment__Basic_3] = parser.NewATNState(Assignment__Basic_3, parser.ATNBasic, true).SetDecision(23) + states[Assignment__Basic_3] = parser.NewATNState(Assignment__Basic_3, parser.ATNBasic, true).SetDecision(39) states[Assignment__BlockEnd] = parser.NewATNState(Assignment__BlockEnd, parser.ATNBlockEnd, true) states[Assignment__Basic_4] = parser.NewATNState(Assignment__Basic_4, parser.ATNBasic, true) states[Assignment__Basic_5] = parser.NewATNState(Assignment__Basic_5, parser.ATNBasic, true) @@ -552,7 +768,7 @@ func BuildATN() *parser.RuntimeATN { states[Assignable__Basic_6] = parser.NewATNState(Assignable__Basic_6, parser.ATNBasic, true) states[Assignable_RightParen] = parser.NewATNState(Assignable_RightParen, parser.ATNBasic, false) states[Assignable__Basic_7] = parser.NewATNState(Assignable__Basic_7, parser.ATNBasic, true) - states[Assignable__Basic_8] = parser.NewATNState(Assignable__Basic_8, parser.ATNBasic, true).SetDecision(24) + states[Assignable__Basic_8] = parser.NewATNState(Assignable__Basic_8, parser.ATNBasic, true).SetDecision(40) states[Assignable__BlockEnd] = parser.NewATNState(Assignable__BlockEnd, parser.ATNBlockEnd, true) states[AssignableWithoutAlts__Basic_0] = parser.NewATNState(AssignableWithoutAlts__Basic_0, parser.ATNBasic, true) states[AssignableWithoutAlts__Basic_1] = parser.NewATNState(AssignableWithoutAlts__Basic_1, parser.ATNBasic, true) @@ -560,21 +776,21 @@ func BuildATN() *parser.RuntimeATN { states[AssignableWithoutAlts__Basic_3] = parser.NewATNState(AssignableWithoutAlts__Basic_3, parser.ATNBasic, true) states[AssignableWithoutAlts__Basic_4] = parser.NewATNState(AssignableWithoutAlts__Basic_4, parser.ATNBasic, true) states[AssignableWithoutAlts__Basic_5] = parser.NewATNState(AssignableWithoutAlts__Basic_5, parser.ATNBasic, true) - states[AssignableWithoutAlts__Basic_6] = parser.NewATNState(AssignableWithoutAlts__Basic_6, parser.ATNBasic, true).SetDecision(25) + states[AssignableWithoutAlts__Basic_6] = parser.NewATNState(AssignableWithoutAlts__Basic_6, parser.ATNBasic, true).SetDecision(41) states[AssignableWithoutAlts__BlockEnd] = parser.NewATNState(AssignableWithoutAlts__BlockEnd, parser.ATNBlockEnd, true) states[AssignableAlternatives__Basic_0] = parser.NewATNState(AssignableAlternatives__Basic_0, parser.ATNBasic, true) states[AssignableAlternatives_Pipe] = parser.NewATNState(AssignableAlternatives_Pipe, parser.ATNBasic, false) states[AssignableAlternatives__Basic_1] = parser.NewATNState(AssignableAlternatives__Basic_1, parser.ATNBasic, true) states[AssignableAlternatives__Basic_2] = parser.NewATNState(AssignableAlternatives__Basic_2, parser.ATNBasic, true) - states[AssignableAlternatives__LoopBack] = parser.NewATNState(AssignableAlternatives__LoopBack, parser.ATNLoopBack, true).SetDecision(26) + states[AssignableAlternatives__LoopBack] = parser.NewATNState(AssignableAlternatives__LoopBack, parser.ATNLoopBack, true).SetDecision(42) states[AssignableAlternatives__LoopEnd] = parser.NewATNState(AssignableAlternatives__LoopEnd, parser.ATNLoopEnd, true) - states[AssignableAlternatives__Basic_3] = parser.NewATNState(AssignableAlternatives__Basic_3, parser.ATNBasic, true).SetDecision(27) + states[AssignableAlternatives__Basic_3] = parser.NewATNState(AssignableAlternatives__Basic_3, parser.ATNBasic, true).SetDecision(43) states[CrossRef_LeftBracket] = parser.NewATNState(CrossRef_LeftBracket, parser.ATNBasic, false) states[CrossRef_Type_ID] = parser.NewATNState(CrossRef_Type_ID, parser.ATNBasic, false) states[CrossRef_Colon] = parser.NewATNState(CrossRef_Colon, parser.ATNBasic, false) states[CrossRef__Basic_0] = parser.NewATNState(CrossRef__Basic_0, parser.ATNBasic, true) states[CrossRef__Basic_1] = parser.NewATNState(CrossRef__Basic_1, parser.ATNBasic, true) - states[CrossRef__Basic_2] = parser.NewATNState(CrossRef__Basic_2, parser.ATNBasic, true).SetDecision(28) + states[CrossRef__Basic_2] = parser.NewATNState(CrossRef__Basic_2, parser.ATNBasic, true).SetDecision(44) states[CrossRef_RightBracket] = parser.NewATNState(CrossRef_RightBracket, parser.ATNBasic, false) states[CrossRef__Basic_3] = parser.NewATNState(CrossRef__Basic_3, parser.ATNBasic, true) states[RuleCall_Rule_ID] = parser.NewATNState(RuleCall_Rule_ID, parser.ATNBasic, false) @@ -587,11 +803,11 @@ func BuildATN() *parser.RuntimeATN { states[Action__Basic_0] = parser.NewATNState(Action__Basic_0, parser.ATNBasic, true) states[Action_Operator_Equals] = parser.NewATNState(Action_Operator_Equals, parser.ATNBasic, false) states[Action__Basic_1] = parser.NewATNState(Action__Basic_1, parser.ATNBasic, true) - states[Action__Basic_2] = parser.NewATNState(Action__Basic_2, parser.ATNBasic, true).SetDecision(29) + states[Action__Basic_2] = parser.NewATNState(Action__Basic_2, parser.ATNBasic, true).SetDecision(45) states[Action__BlockEnd] = parser.NewATNState(Action__BlockEnd, parser.ATNBlockEnd, true) states[Action_current] = parser.NewATNState(Action_current, parser.ATNBasic, false) states[Action__Basic_3] = parser.NewATNState(Action__Basic_3, parser.ATNBasic, true) - states[Action__Basic_4] = parser.NewATNState(Action__Basic_4, parser.ATNBasic, true).SetDecision(30) + states[Action__Basic_4] = parser.NewATNState(Action__Basic_4, parser.ATNBasic, true).SetDecision(46) states[Action_RightBrace] = parser.NewATNState(Action_RightBrace, parser.ATNBasic, false) states[Action__Basic_5] = parser.NewATNState(Action__Basic_5, parser.ATNBasic, true) states[CompositeRule_composite] = parser.NewATNState(CompositeRule_composite, parser.ATNBasic, false) @@ -600,20 +816,20 @@ func BuildATN() *parser.RuntimeATN { states[CompositeRule__Basic_0] = parser.NewATNState(CompositeRule__Basic_0, parser.ATNBasic, true) states[CompositeRule_Semicolon] = parser.NewATNState(CompositeRule_Semicolon, parser.ATNBasic, false) states[CompositeRule__Basic_1] = parser.NewATNState(CompositeRule__Basic_1, parser.ATNBasic, true) - states[CompositeRule__Basic_2] = parser.NewATNState(CompositeRule__Basic_2, parser.ATNBasic, true).SetDecision(31) + states[CompositeRule__Basic_2] = parser.NewATNState(CompositeRule__Basic_2, parser.ATNBasic, true).SetDecision(47) states[CompositeAlternatives__Basic_0] = parser.NewATNState(CompositeAlternatives__Basic_0, parser.ATNBasic, true) states[CompositeAlternatives_Pipe] = parser.NewATNState(CompositeAlternatives_Pipe, parser.ATNBasic, false) states[CompositeAlternatives__Basic_1] = parser.NewATNState(CompositeAlternatives__Basic_1, parser.ATNBasic, true) states[CompositeAlternatives__Basic_2] = parser.NewATNState(CompositeAlternatives__Basic_2, parser.ATNBasic, true) - states[CompositeAlternatives__LoopBack] = parser.NewATNState(CompositeAlternatives__LoopBack, parser.ATNLoopBack, true).SetDecision(32) + states[CompositeAlternatives__LoopBack] = parser.NewATNState(CompositeAlternatives__LoopBack, parser.ATNLoopBack, true).SetDecision(48) states[CompositeAlternatives__LoopEnd] = parser.NewATNState(CompositeAlternatives__LoopEnd, parser.ATNLoopEnd, true) - states[CompositeAlternatives__Basic_3] = parser.NewATNState(CompositeAlternatives__Basic_3, parser.ATNBasic, true).SetDecision(33) + states[CompositeAlternatives__Basic_3] = parser.NewATNState(CompositeAlternatives__Basic_3, parser.ATNBasic, true).SetDecision(49) states[CompositeGroup__Basic_0] = parser.NewATNState(CompositeGroup__Basic_0, parser.ATNBasic, true) states[CompositeGroup__Basic_1] = parser.NewATNState(CompositeGroup__Basic_1, parser.ATNBasic, true) states[CompositeGroup__Basic_2] = parser.NewATNState(CompositeGroup__Basic_2, parser.ATNBasic, true) - states[CompositeGroup__LoopBack] = parser.NewATNState(CompositeGroup__LoopBack, parser.ATNLoopBack, true).SetDecision(34) + states[CompositeGroup__LoopBack] = parser.NewATNState(CompositeGroup__LoopBack, parser.ATNLoopBack, true).SetDecision(50) states[CompositeGroup__LoopEnd] = parser.NewATNState(CompositeGroup__LoopEnd, parser.ATNLoopEnd, true) - states[CompositeGroup__Basic_3] = parser.NewATNState(CompositeGroup__Basic_3, parser.ATNBasic, true).SetDecision(35) + states[CompositeGroup__Basic_3] = parser.NewATNState(CompositeGroup__Basic_3, parser.ATNBasic, true).SetDecision(51) states[CompositeElement__Basic_0] = parser.NewATNState(CompositeElement__Basic_0, parser.ATNBasic, true) states[CompositeElement__Basic_1] = parser.NewATNState(CompositeElement__Basic_1, parser.ATNBasic, true) states[CompositeElement__Basic_2] = parser.NewATNState(CompositeElement__Basic_2, parser.ATNBasic, true) @@ -622,17 +838,11 @@ func BuildATN() *parser.RuntimeATN { states[CompositeElement__Basic_4] = parser.NewATNState(CompositeElement__Basic_4, parser.ATNBasic, true) states[CompositeElement_RightParen] = parser.NewATNState(CompositeElement_RightParen, parser.ATNBasic, false) states[CompositeElement__Basic_5] = parser.NewATNState(CompositeElement__Basic_5, parser.ATNBasic, true) - states[CompositeElement__Basic_6] = parser.NewATNState(CompositeElement__Basic_6, parser.ATNBasic, true).SetDecision(36) - states[CompositeElement__BlockEnd_0] = parser.NewATNState(CompositeElement__BlockEnd_0, parser.ATNBlockEnd, true) - states[CompositeElement_Cardinality_Asterisk] = parser.NewATNState(CompositeElement_Cardinality_Asterisk, parser.ATNBasic, false) - states[CompositeElement__Basic_7] = parser.NewATNState(CompositeElement__Basic_7, parser.ATNBasic, true) - states[CompositeElement_Cardinality_Plus] = parser.NewATNState(CompositeElement_Cardinality_Plus, parser.ATNBasic, false) + states[CompositeElement__Basic_6] = parser.NewATNState(CompositeElement__Basic_6, parser.ATNBasic, true).SetDecision(52) + states[CompositeElement__BlockEnd] = parser.NewATNState(CompositeElement__BlockEnd, parser.ATNBlockEnd, true) + states[CompositeElement__Basic_7] = parser.NewATNState(CompositeElement__Basic_7, parser.ATNBasic, false) states[CompositeElement__Basic_8] = parser.NewATNState(CompositeElement__Basic_8, parser.ATNBasic, true) - states[CompositeElement_Cardinality_Question] = parser.NewATNState(CompositeElement_Cardinality_Question, parser.ATNBasic, false) - states[CompositeElement__Basic_9] = parser.NewATNState(CompositeElement__Basic_9, parser.ATNBasic, true) - states[CompositeElement__Basic_10] = parser.NewATNState(CompositeElement__Basic_10, parser.ATNBasic, true).SetDecision(37) - states[CompositeElement__BlockEnd_1] = parser.NewATNState(CompositeElement__BlockEnd_1, parser.ATNBlockEnd, true) - states[CompositeElement__Basic_11] = parser.NewATNState(CompositeElement__Basic_11, parser.ATNBasic, true).SetDecision(38) + states[CompositeElement__Basic_9] = parser.NewATNState(CompositeElement__Basic_9, parser.ATNBasic, true).SetDecision(53) states[Grammar__Start].AppendTransitions( parser.NewEpsilonTransition(states[Grammar_grammar]), ) @@ -660,11 +870,44 @@ func BuildATN() *parser.RuntimeATN { states[ParserRule__Start].AppendTransitions( parser.NewEpsilonTransition(states[ParserRule__Basic_1]), ) - states[Token__Start].AppendTransitions( - parser.NewEpsilonTransition(states[Token__Basic_3]), + states[TokenDecl__Start].AppendTransitions( + parser.NewEpsilonTransition(states[TokenDecl__Basic_2]), + ) + states[TokenContent__Start].AppendTransitions( + parser.NewEpsilonTransition(states[TokenContent__Basic_4]), + ) + states[RegexpTokenContent__Start].AppendTransitions( + parser.NewEpsilonTransition(states[RegexpTokenContent_Regexp_RegexLiteral]), + ) + states[KeywordTokenContent__Start].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordTokenContent__Basic_0]), + ) + states[TokenCommand__Start].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand_DashGreaterThan]), ) states[TokenGroup__Start].AppendTransitions( - parser.NewEpsilonTransition(states[TokenGroup_token]), + parser.NewEpsilonTransition(states[TokenGroup__Basic_2]), + ) + states[TokenMode__Start].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode_token]), + ) + states[TokenModeMember__Start].AppendTransitions( + parser.NewEpsilonTransition(states[TokenModeMember__Basic_10]), + ) + states[TokenDeclUsage__Start].AppendTransitions( + parser.NewEpsilonTransition(states[TokenDeclUsage__Basic_0]), + ) + states[TokenGroupUsage__Start].AppendTransitions( + parser.NewEpsilonTransition(states[TokenGroupUsage__Basic_0]), + ) + states[TokenUsage__Start].AppendTransitions( + parser.NewEpsilonTransition(states[TokenUsage__Basic_2]), + ) + states[KeywordUsage__Start].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordUsage__Basic_2]), + ) + states[KeywordSelector__Start].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordSelector_keywords]), ) states[Alternatives__Start].AppendTransitions( parser.NewEpsilonTransition(states[Alternatives__Basic_0]), @@ -734,7 +977,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[Grammar__BlockEnd]), ) states[Grammar__Basic_4].AppendTransitions( - parser.NewRuleTransition(states[Token__Start], states[Grammar__Basic_5], nil), + parser.NewRuleTransition(states[TokenDecl__Start], states[Grammar__Basic_5], nil), ) states[Grammar__Basic_5].AppendTransitions( parser.NewEpsilonTransition(states[Grammar__BlockEnd]), @@ -746,29 +989,36 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[Grammar__BlockEnd]), ) states[Grammar__Basic_8].AppendTransitions( - parser.NewRuleTransition(states[Interface__Start], states[Grammar__Basic_9], nil), + parser.NewRuleTransition(states[TokenMode__Start], states[Grammar__Basic_9], nil), ) states[Grammar__Basic_9].AppendTransitions( parser.NewEpsilonTransition(states[Grammar__BlockEnd]), ) states[Grammar__Basic_10].AppendTransitions( - parser.NewRuleTransition(states[CompositeRule__Start], states[Grammar__Basic_11], nil), + parser.NewRuleTransition(states[Interface__Start], states[Grammar__Basic_11], nil), ) states[Grammar__Basic_11].AppendTransitions( parser.NewEpsilonTransition(states[Grammar__BlockEnd]), ) states[Grammar__Basic_12].AppendTransitions( + parser.NewRuleTransition(states[CompositeRule__Start], states[Grammar__Basic_13], nil), + ) + states[Grammar__Basic_13].AppendTransitions( + parser.NewEpsilonTransition(states[Grammar__BlockEnd]), + ) + states[Grammar__Basic_14].AppendTransitions( parser.NewEpsilonTransition(states[Grammar__Basic_2]), parser.NewEpsilonTransition(states[Grammar__Basic_4]), parser.NewEpsilonTransition(states[Grammar__Basic_6]), parser.NewEpsilonTransition(states[Grammar__Basic_8]), parser.NewEpsilonTransition(states[Grammar__Basic_10]), + parser.NewEpsilonTransition(states[Grammar__Basic_12]), ) states[Grammar__BlockEnd].AppendTransitions( parser.NewEpsilonTransition(states[Grammar__LoopBack]), ) states[Grammar__LoopEntry].AppendTransitions( - parser.NewEpsilonTransition(states[Grammar__Basic_12]), + parser.NewEpsilonTransition(states[Grammar__Basic_14]), parser.NewEpsilonTransition(states[Grammar__LoopEnd]), ) states[Grammar__LoopEnd].AppendTransitions( @@ -974,50 +1224,149 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[ParserRule_Semicolon]), parser.NewEpsilonTransition(states[ParserRule__Basic_5]), ) - states[Token_Type_hidden].AppendTransitions( - parser.NewAtomTransition(states[Token__Basic_0], Keyword_hidden, nil), + states[TokenDecl__Basic_0].AppendTransitions( + parser.NewAtomTransition(states[TokenDecl__Basic_1], TokenGroup_TokenModifier, nil), + ) + states[TokenDecl__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenDecl_token]), + ) + states[TokenDecl__Basic_2].AppendTransitions( + parser.NewEpsilonTransition(states[TokenDecl__Basic_0]), + parser.NewEpsilonTransition(states[TokenDecl__Basic_1]), + ) + states[TokenDecl_token].AppendTransitions( + parser.NewAtomTransition(states[TokenDecl_Name_ID], Keyword_token, nil), + ) + states[TokenDecl_Name_ID].AppendTransitions( + parser.NewAtomTransition(states[TokenDecl_Colon], Token_ID, nil), + ) + states[TokenDecl_Colon].AppendTransitions( + parser.NewAtomTransition(states[TokenDecl__Basic_3], Keyword_Colon, nil), + ) + states[TokenDecl__Basic_3].AppendTransitions( + parser.NewRuleTransition(states[TokenContent__Start], states[TokenDecl__Basic_6], nil), + ) + states[TokenDecl__Basic_4].AppendTransitions( + parser.NewRuleTransition(states[TokenCommand__Start], states[TokenDecl__Basic_5], nil), ) - states[Token__Basic_0].AppendTransitions( - parser.NewEpsilonTransition(states[Token__BlockEnd]), + states[TokenDecl__Basic_5].AppendTransitions( + parser.NewEpsilonTransition(states[TokenDecl__Basic_8]), ) - states[Token_Type_comment].AppendTransitions( - parser.NewAtomTransition(states[Token__Basic_1], Keyword_comment, nil), + states[TokenDecl__Basic_6].AppendTransitions( + parser.NewEpsilonTransition(states[TokenDecl__Basic_4]), + parser.NewEpsilonTransition(states[TokenDecl__Basic_5]), ) - states[Token__Basic_1].AppendTransitions( - parser.NewEpsilonTransition(states[Token__BlockEnd]), + states[TokenDecl_Semicolon].AppendTransitions( + parser.NewAtomTransition(states[TokenDecl__Basic_7], Keyword_Semicolon, nil), ) - states[Token__Basic_2].AppendTransitions( - parser.NewEpsilonTransition(states[Token_Type_hidden]), - parser.NewEpsilonTransition(states[Token_Type_comment]), + states[TokenDecl__Basic_7].AppendTransitions( + parser.NewEpsilonTransition(states[TokenDecl__Stop]), ) - states[Token__BlockEnd].AppendTransitions( - parser.NewEpsilonTransition(states[Token_token]), + states[TokenDecl__Basic_8].AppendTransitions( + parser.NewEpsilonTransition(states[TokenDecl_Semicolon]), + parser.NewEpsilonTransition(states[TokenDecl__Basic_7]), ) - states[Token__Basic_3].AppendTransitions( - parser.NewEpsilonTransition(states[Token__Basic_2]), - parser.NewEpsilonTransition(states[Token__BlockEnd]), + states[TokenContent__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[RegexpTokenContent__Start], states[TokenContent__Basic_1], nil), ) - states[Token_token].AppendTransitions( - parser.NewAtomTransition(states[Token_Name_ID], Keyword_token, nil), + states[TokenContent__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenContent__BlockEnd]), ) - states[Token_Name_ID].AppendTransitions( - parser.NewAtomTransition(states[Token_Colon], Token_ID, nil), + states[TokenContent__Basic_2].AppendTransitions( + parser.NewRuleTransition(states[KeywordTokenContent__Start], states[TokenContent__Basic_3], nil), ) - states[Token_Colon].AppendTransitions( - parser.NewAtomTransition(states[Token_Regexp_RegexLiteral], Keyword_Colon, nil), + states[TokenContent__Basic_3].AppendTransitions( + parser.NewEpsilonTransition(states[TokenContent__BlockEnd]), ) - states[Token_Regexp_RegexLiteral].AppendTransitions( - parser.NewAtomTransition(states[Token__Basic_5], Token_RegexLiteral, nil), + states[TokenContent__Basic_4].AppendTransitions( + parser.NewEpsilonTransition(states[TokenContent__Basic_0]), + parser.NewEpsilonTransition(states[TokenContent__Basic_2]), ) - states[Token_Semicolon].AppendTransitions( - parser.NewAtomTransition(states[Token__Basic_4], Keyword_Semicolon, nil), + states[TokenContent__BlockEnd].AppendTransitions( + parser.NewEpsilonTransition(states[TokenContent__Stop]), ) - states[Token__Basic_4].AppendTransitions( - parser.NewEpsilonTransition(states[Token__Stop]), + states[RegexpTokenContent_Regexp_RegexLiteral].AppendTransitions( + parser.NewAtomTransition(states[RegexpTokenContent__Basic], Token_RegexLiteral, nil), ) - states[Token__Basic_5].AppendTransitions( - parser.NewEpsilonTransition(states[Token_Semicolon]), - parser.NewEpsilonTransition(states[Token__Basic_4]), + states[RegexpTokenContent__Basic].AppendTransitions( + parser.NewEpsilonTransition(states[RegexpTokenContent__Stop]), + ) + states[KeywordTokenContent__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[Keyword__Start], states[KeywordTokenContent__Basic_1], nil), + ) + states[KeywordTokenContent__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordTokenContent__Stop]), + ) + states[TokenCommand_DashGreaterThan].AppendTransitions( + parser.NewAtomTransition(states[TokenCommand__Basic_3], Keyword_DashGreaterThan, nil), + ) + states[TokenCommand_Type_push].AppendTransitions( + parser.NewAtomTransition(states[TokenCommand__Basic_0], Keyword_push, nil), + ) + states[TokenCommand__Basic_0].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand__BlockEnd_0]), + ) + states[TokenCommand_Type_pop].AppendTransitions( + parser.NewAtomTransition(states[TokenCommand__Basic_1], Keyword_pop, nil), + ) + states[TokenCommand__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand__BlockEnd_0]), + ) + states[TokenCommand_Type_mode].AppendTransitions( + parser.NewAtomTransition(states[TokenCommand__Basic_2], Keyword_mode, nil), + ) + states[TokenCommand__Basic_2].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand__BlockEnd_0]), + ) + states[TokenCommand__Basic_3].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand_Type_push]), + parser.NewEpsilonTransition(states[TokenCommand_Type_pop]), + parser.NewEpsilonTransition(states[TokenCommand_Type_mode]), + ) + states[TokenCommand__BlockEnd_0].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand__Basic_8]), + ) + states[TokenCommand_LeftParen].AppendTransitions( + parser.NewAtomTransition(states[TokenCommand__Basic_6], Keyword_LeftParen, nil), + ) + states[TokenCommand_Mode_ID].AppendTransitions( + parser.NewAtomTransition(states[TokenCommand__Basic_4], Token_ID, &parser.CompletionHint{Field: "TokenCommand.Mode"}), + ) + states[TokenCommand__Basic_4].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand__BlockEnd_1]), + ) + states[TokenCommand_Default_default].AppendTransitions( + parser.NewAtomTransition(states[TokenCommand__Basic_5], Keyword_default, nil), + ) + states[TokenCommand__Basic_5].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand__BlockEnd_1]), + ) + states[TokenCommand__Basic_6].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand_Mode_ID]), + parser.NewEpsilonTransition(states[TokenCommand_Default_default]), + ) + states[TokenCommand__BlockEnd_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand_RightParen]), + ) + states[TokenCommand_RightParen].AppendTransitions( + parser.NewAtomTransition(states[TokenCommand__Basic_7], Keyword_RightParen, nil), + ) + states[TokenCommand__Basic_7].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand__Stop]), + ) + states[TokenCommand__Basic_8].AppendTransitions( + parser.NewEpsilonTransition(states[TokenCommand_LeftParen]), + parser.NewEpsilonTransition(states[TokenCommand__Basic_7]), + ) + states[TokenGroup__Basic_0].AppendTransitions( + parser.NewAtomTransition(states[TokenGroup__Basic_1], TokenGroup_TokenModifier, nil), + ) + states[TokenGroup__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenGroup_token]), + ) + states[TokenGroup__Basic_2].AppendTransitions( + parser.NewEpsilonTransition(states[TokenGroup__Basic_0]), + parser.NewEpsilonTransition(states[TokenGroup__Basic_1]), ) states[TokenGroup_token].AppendTransitions( parser.NewAtomTransition(states[TokenGroup_group], Keyword_token, nil), @@ -1032,36 +1381,36 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[TokenGroup__LoopEntry], Keyword_LeftBrace, nil), ) states[TokenGroup_TokenRefs_ID].AppendTransitions( - parser.NewAtomTransition(states[TokenGroup__Basic_0], Token_ID, &parser.CompletionHint{Field: "TokenGroup.TokenRefs"}), + parser.NewAtomTransition(states[TokenGroup__Basic_3], Token_ID, &parser.CompletionHint{Field: "TokenGroup.TokenRefs"}), ) - states[TokenGroup__Basic_0].AppendTransitions( + states[TokenGroup__Basic_3].AppendTransitions( parser.NewEpsilonTransition(states[TokenGroup__BlockEnd]), ) - states[TokenGroup_keywords].AppendTransitions( - parser.NewAtomTransition(states[TokenGroup_Regexps_RegexLiteral], Keyword_keywords, nil), - ) - states[TokenGroup_Regexps_RegexLiteral].AppendTransitions( - parser.NewAtomTransition(states[TokenGroup__Basic_1], Token_RegexLiteral, nil), + states[TokenGroup__Basic_4].AppendTransitions( + parser.NewRuleTransition(states[Keyword__Start], states[TokenGroup__Basic_5], nil), ) - states[TokenGroup__Basic_1].AppendTransitions( + states[TokenGroup__Basic_5].AppendTransitions( parser.NewEpsilonTransition(states[TokenGroup__BlockEnd]), ) - states[TokenGroup__Basic_2].AppendTransitions( - parser.NewRuleTransition(states[Keyword__Start], states[TokenGroup__Basic_3], nil), + states[TokenGroup_keywords].AppendTransitions( + parser.NewAtomTransition(states[TokenGroup_KeywordSelectors_RegexLiteral], Keyword_keywords, nil), ) - states[TokenGroup__Basic_3].AppendTransitions( + states[TokenGroup_KeywordSelectors_RegexLiteral].AppendTransitions( + parser.NewAtomTransition(states[TokenGroup__Basic_6], Token_RegexLiteral, nil), + ) + states[TokenGroup__Basic_6].AppendTransitions( parser.NewEpsilonTransition(states[TokenGroup__BlockEnd]), ) - states[TokenGroup__Basic_4].AppendTransitions( + states[TokenGroup__Basic_7].AppendTransitions( parser.NewEpsilonTransition(states[TokenGroup_TokenRefs_ID]), + parser.NewEpsilonTransition(states[TokenGroup__Basic_4]), parser.NewEpsilonTransition(states[TokenGroup_keywords]), - parser.NewEpsilonTransition(states[TokenGroup__Basic_2]), ) states[TokenGroup__BlockEnd].AppendTransitions( parser.NewEpsilonTransition(states[TokenGroup__LoopBack]), ) states[TokenGroup__LoopEntry].AppendTransitions( - parser.NewEpsilonTransition(states[TokenGroup__Basic_4]), + parser.NewEpsilonTransition(states[TokenGroup__Basic_7]), parser.NewEpsilonTransition(states[TokenGroup__LoopEnd]), ) states[TokenGroup__LoopEnd].AppendTransitions( @@ -1071,11 +1420,212 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[TokenGroup__LoopEntry]), ) states[TokenGroup_RightBrace].AppendTransitions( - parser.NewAtomTransition(states[TokenGroup__Basic_5], Keyword_RightBrace, nil), + parser.NewAtomTransition(states[TokenGroup__Basic_10], Keyword_RightBrace, nil), ) - states[TokenGroup__Basic_5].AppendTransitions( + states[TokenGroup__Basic_8].AppendTransitions( + parser.NewRuleTransition(states[TokenCommand__Start], states[TokenGroup__Basic_9], nil), + ) + states[TokenGroup__Basic_9].AppendTransitions( + parser.NewEpsilonTransition(states[TokenGroup__Basic_12]), + ) + states[TokenGroup__Basic_10].AppendTransitions( + parser.NewEpsilonTransition(states[TokenGroup__Basic_8]), + parser.NewEpsilonTransition(states[TokenGroup__Basic_9]), + ) + states[TokenGroup_Semicolon].AppendTransitions( + parser.NewAtomTransition(states[TokenGroup__Basic_11], Keyword_Semicolon, nil), + ) + states[TokenGroup__Basic_11].AppendTransitions( parser.NewEpsilonTransition(states[TokenGroup__Stop]), ) + states[TokenGroup__Basic_12].AppendTransitions( + parser.NewEpsilonTransition(states[TokenGroup_Semicolon]), + parser.NewEpsilonTransition(states[TokenGroup__Basic_11]), + ) + states[TokenMode_token].AppendTransitions( + parser.NewAtomTransition(states[TokenMode_mode], Keyword_token, nil), + ) + states[TokenMode_mode].AppendTransitions( + parser.NewAtomTransition(states[TokenMode__Basic_2], Keyword_mode, nil), + ) + states[TokenMode_Name_ID].AppendTransitions( + parser.NewAtomTransition(states[TokenMode__Basic_0], Token_ID, nil), + ) + states[TokenMode__Basic_0].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode__BlockEnd]), + ) + states[TokenMode_Default_default].AppendTransitions( + parser.NewAtomTransition(states[TokenMode__Basic_1], Keyword_default, nil), + ) + states[TokenMode__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode__BlockEnd]), + ) + states[TokenMode__Basic_2].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode_Name_ID]), + parser.NewEpsilonTransition(states[TokenMode_Default_default]), + ) + states[TokenMode__BlockEnd].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode_LeftBrace]), + ) + states[TokenMode_LeftBrace].AppendTransitions( + parser.NewAtomTransition(states[TokenMode__LoopEntry], Keyword_LeftBrace, nil), + ) + states[TokenMode__Basic_3].AppendTransitions( + parser.NewRuleTransition(states[TokenModeMember__Start], states[TokenMode__Basic_4], nil), + ) + states[TokenMode__Basic_4].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode__LoopBack]), + ) + states[TokenMode__LoopEntry].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode__Basic_3]), + parser.NewEpsilonTransition(states[TokenMode__LoopEnd]), + ) + states[TokenMode__LoopEnd].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode_RightBrace]), + ) + states[TokenMode__LoopBack].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode__LoopEntry]), + ) + states[TokenMode_RightBrace].AppendTransitions( + parser.NewAtomTransition(states[TokenMode__Basic_5], Keyword_RightBrace, nil), + ) + states[TokenMode__Basic_5].AppendTransitions( + parser.NewEpsilonTransition(states[TokenMode__Stop]), + ) + states[TokenModeMember__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[TokenDeclUsage__Start], states[TokenModeMember__Basic_1], nil), + ) + states[TokenModeMember__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenModeMember__BlockEnd]), + ) + states[TokenModeMember__Basic_2].AppendTransitions( + parser.NewRuleTransition(states[TokenGroupUsage__Start], states[TokenModeMember__Basic_3], nil), + ) + states[TokenModeMember__Basic_3].AppendTransitions( + parser.NewEpsilonTransition(states[TokenModeMember__BlockEnd]), + ) + states[TokenModeMember__Basic_4].AppendTransitions( + parser.NewRuleTransition(states[TokenUsage__Start], states[TokenModeMember__Basic_5], nil), + ) + states[TokenModeMember__Basic_5].AppendTransitions( + parser.NewEpsilonTransition(states[TokenModeMember__BlockEnd]), + ) + states[TokenModeMember__Basic_6].AppendTransitions( + parser.NewRuleTransition(states[KeywordUsage__Start], states[TokenModeMember__Basic_7], nil), + ) + states[TokenModeMember__Basic_7].AppendTransitions( + parser.NewEpsilonTransition(states[TokenModeMember__BlockEnd]), + ) + states[TokenModeMember__Basic_8].AppendTransitions( + parser.NewRuleTransition(states[KeywordSelector__Start], states[TokenModeMember__Basic_9], nil), + ) + states[TokenModeMember__Basic_9].AppendTransitions( + parser.NewEpsilonTransition(states[TokenModeMember__BlockEnd]), + ) + states[TokenModeMember__Basic_10].AppendTransitions( + parser.NewEpsilonTransition(states[TokenModeMember__Basic_0]), + parser.NewEpsilonTransition(states[TokenModeMember__Basic_2]), + parser.NewEpsilonTransition(states[TokenModeMember__Basic_4]), + parser.NewEpsilonTransition(states[TokenModeMember__Basic_6]), + parser.NewEpsilonTransition(states[TokenModeMember__Basic_8]), + ) + states[TokenModeMember__BlockEnd].AppendTransitions( + parser.NewEpsilonTransition(states[TokenModeMember__Stop]), + ) + states[TokenDeclUsage__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[TokenDecl__Start], states[TokenDeclUsage__Basic_1], nil), + ) + states[TokenDeclUsage__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenDeclUsage__Stop]), + ) + states[TokenGroupUsage__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[TokenGroup__Start], states[TokenGroupUsage__Basic_1], nil), + ) + states[TokenGroupUsage__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenGroupUsage__Stop]), + ) + states[TokenUsage__Basic_0].AppendTransitions( + parser.NewAtomTransition(states[TokenUsage__Basic_1], TokenGroup_TokenModifier, nil), + ) + states[TokenUsage__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[TokenUsage_TokenRef_ID]), + ) + states[TokenUsage__Basic_2].AppendTransitions( + parser.NewEpsilonTransition(states[TokenUsage__Basic_0]), + parser.NewEpsilonTransition(states[TokenUsage__Basic_1]), + ) + states[TokenUsage_TokenRef_ID].AppendTransitions( + parser.NewAtomTransition(states[TokenUsage__Basic_5], Token_ID, &parser.CompletionHint{Field: "TokenUsage.TokenRef"}), + ) + states[TokenUsage__Basic_3].AppendTransitions( + parser.NewRuleTransition(states[TokenCommand__Start], states[TokenUsage__Basic_4], nil), + ) + states[TokenUsage__Basic_4].AppendTransitions( + parser.NewEpsilonTransition(states[TokenUsage__Basic_7]), + ) + states[TokenUsage__Basic_5].AppendTransitions( + parser.NewEpsilonTransition(states[TokenUsage__Basic_3]), + parser.NewEpsilonTransition(states[TokenUsage__Basic_4]), + ) + states[TokenUsage_Semicolon].AppendTransitions( + parser.NewAtomTransition(states[TokenUsage__Basic_6], Keyword_Semicolon, nil), + ) + states[TokenUsage__Basic_6].AppendTransitions( + parser.NewEpsilonTransition(states[TokenUsage__Stop]), + ) + states[TokenUsage__Basic_7].AppendTransitions( + parser.NewEpsilonTransition(states[TokenUsage_Semicolon]), + parser.NewEpsilonTransition(states[TokenUsage__Basic_6]), + ) + states[KeywordUsage__Basic_0].AppendTransitions( + parser.NewAtomTransition(states[KeywordUsage__Basic_1], TokenGroup_TokenModifier, nil), + ) + states[KeywordUsage__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordUsage__Basic_3]), + ) + states[KeywordUsage__Basic_2].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordUsage__Basic_0]), + parser.NewEpsilonTransition(states[KeywordUsage__Basic_1]), + ) + states[KeywordUsage__Basic_3].AppendTransitions( + parser.NewRuleTransition(states[Keyword__Start], states[KeywordUsage__Basic_6], nil), + ) + states[KeywordUsage__Basic_4].AppendTransitions( + parser.NewRuleTransition(states[TokenCommand__Start], states[KeywordUsage__Basic_5], nil), + ) + states[KeywordUsage__Basic_5].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordUsage__Basic_8]), + ) + states[KeywordUsage__Basic_6].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordUsage__Basic_4]), + parser.NewEpsilonTransition(states[KeywordUsage__Basic_5]), + ) + states[KeywordUsage_Semicolon].AppendTransitions( + parser.NewAtomTransition(states[KeywordUsage__Basic_7], Keyword_Semicolon, nil), + ) + states[KeywordUsage__Basic_7].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordUsage__Stop]), + ) + states[KeywordUsage__Basic_8].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordUsage_Semicolon]), + parser.NewEpsilonTransition(states[KeywordUsage__Basic_7]), + ) + states[KeywordSelector_keywords].AppendTransitions( + parser.NewAtomTransition(states[KeywordSelector_Selector_RegexLiteral], Keyword_keywords, nil), + ) + states[KeywordSelector_Selector_RegexLiteral].AppendTransitions( + parser.NewAtomTransition(states[KeywordSelector__Basic_1], Token_RegexLiteral, nil), + ) + states[KeywordSelector_Semicolon].AppendTransitions( + parser.NewAtomTransition(states[KeywordSelector__Basic_0], Keyword_Semicolon, nil), + ) + states[KeywordSelector__Basic_0].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordSelector__Stop]), + ) + states[KeywordSelector__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[KeywordSelector_Semicolon]), + parser.NewEpsilonTransition(states[KeywordSelector__Basic_0]), + ) states[Alternatives__Basic_0].AppendTransitions( parser.NewRuleTransition(states[Group__Start], states[Alternatives__Basic_3], nil), ) @@ -1123,25 +1673,25 @@ func BuildATN() *parser.RuntimeATN { parser.NewRuleTransition(states[Keyword__Start], states[Element__Basic_1], nil), ) states[Element__Basic_1].AppendTransitions( - parser.NewEpsilonTransition(states[Element__BlockEnd_0]), + parser.NewEpsilonTransition(states[Element__BlockEnd]), ) states[Element__Basic_2].AppendTransitions( parser.NewRuleTransition(states[Assignment__Start], states[Element__Basic_3], nil), ) states[Element__Basic_3].AppendTransitions( - parser.NewEpsilonTransition(states[Element__BlockEnd_0]), + parser.NewEpsilonTransition(states[Element__BlockEnd]), ) states[Element__Basic_4].AppendTransitions( parser.NewRuleTransition(states[RuleCall__Start], states[Element__Basic_5], nil), ) states[Element__Basic_5].AppendTransitions( - parser.NewEpsilonTransition(states[Element__BlockEnd_0]), + parser.NewEpsilonTransition(states[Element__BlockEnd]), ) states[Element__Basic_6].AppendTransitions( parser.NewRuleTransition(states[Action__Start], states[Element__Basic_7], nil), ) states[Element__Basic_7].AppendTransitions( - parser.NewEpsilonTransition(states[Element__BlockEnd_0]), + parser.NewEpsilonTransition(states[Element__BlockEnd]), ) states[Element_LeftParen].AppendTransitions( parser.NewAtomTransition(states[Element__Basic_8], Keyword_LeftParen, nil), @@ -1153,7 +1703,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[Element__Basic_9], Keyword_RightParen, nil), ) states[Element__Basic_9].AppendTransitions( - parser.NewEpsilonTransition(states[Element__BlockEnd_0]), + parser.NewEpsilonTransition(states[Element__BlockEnd]), ) states[Element__Basic_10].AppendTransitions( parser.NewEpsilonTransition(states[Element__Basic_0]), @@ -1162,38 +1712,18 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[Element__Basic_6]), parser.NewEpsilonTransition(states[Element_LeftParen]), ) - states[Element__BlockEnd_0].AppendTransitions( - parser.NewEpsilonTransition(states[Element__Basic_15]), - ) - states[Element_Cardinality_Asterisk].AppendTransitions( - parser.NewAtomTransition(states[Element__Basic_11], Keyword_Asterisk, nil), + states[Element__BlockEnd].AppendTransitions( + parser.NewEpsilonTransition(states[Element__Basic_13]), ) states[Element__Basic_11].AppendTransitions( - parser.NewEpsilonTransition(states[Element__BlockEnd_1]), - ) - states[Element_Cardinality_Plus].AppendTransitions( - parser.NewAtomTransition(states[Element__Basic_12], Keyword_Plus, nil), + parser.NewAtomTransition(states[Element__Basic_12], TokenGroup_Cardinality, nil), ) states[Element__Basic_12].AppendTransitions( - parser.NewEpsilonTransition(states[Element__BlockEnd_1]), - ) - states[Element_Cardinality_Question].AppendTransitions( - parser.NewAtomTransition(states[Element__Basic_13], Keyword_Question, nil), - ) - states[Element__Basic_13].AppendTransitions( - parser.NewEpsilonTransition(states[Element__BlockEnd_1]), - ) - states[Element__Basic_14].AppendTransitions( - parser.NewEpsilonTransition(states[Element_Cardinality_Asterisk]), - parser.NewEpsilonTransition(states[Element_Cardinality_Plus]), - parser.NewEpsilonTransition(states[Element_Cardinality_Question]), - ) - states[Element__BlockEnd_1].AppendTransitions( parser.NewEpsilonTransition(states[Element__Stop]), ) - states[Element__Basic_15].AppendTransitions( - parser.NewEpsilonTransition(states[Element__Basic_14]), - parser.NewEpsilonTransition(states[Element__BlockEnd_1]), + states[Element__Basic_13].AppendTransitions( + parser.NewEpsilonTransition(states[Element__Basic_11]), + parser.NewEpsilonTransition(states[Element__Basic_12]), ) states[Keyword_Value_StringLiteral].AppendTransitions( parser.NewAtomTransition(states[Keyword__Basic], Token_StringLiteral, nil), @@ -1471,13 +2001,13 @@ func BuildATN() *parser.RuntimeATN { parser.NewRuleTransition(states[Keyword__Start], states[CompositeElement__Basic_1], nil), ) states[CompositeElement__Basic_1].AppendTransitions( - parser.NewEpsilonTransition(states[CompositeElement__BlockEnd_0]), + parser.NewEpsilonTransition(states[CompositeElement__BlockEnd]), ) states[CompositeElement__Basic_2].AppendTransitions( parser.NewRuleTransition(states[RuleCall__Start], states[CompositeElement__Basic_3], nil), ) states[CompositeElement__Basic_3].AppendTransitions( - parser.NewEpsilonTransition(states[CompositeElement__BlockEnd_0]), + parser.NewEpsilonTransition(states[CompositeElement__BlockEnd]), ) states[CompositeElement_LeftParen].AppendTransitions( parser.NewAtomTransition(states[CompositeElement__Basic_4], Keyword_LeftParen, nil), @@ -1489,49 +2019,29 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[CompositeElement__Basic_5], Keyword_RightParen, nil), ) states[CompositeElement__Basic_5].AppendTransitions( - parser.NewEpsilonTransition(states[CompositeElement__BlockEnd_0]), + parser.NewEpsilonTransition(states[CompositeElement__BlockEnd]), ) states[CompositeElement__Basic_6].AppendTransitions( parser.NewEpsilonTransition(states[CompositeElement__Basic_0]), parser.NewEpsilonTransition(states[CompositeElement__Basic_2]), parser.NewEpsilonTransition(states[CompositeElement_LeftParen]), ) - states[CompositeElement__BlockEnd_0].AppendTransitions( - parser.NewEpsilonTransition(states[CompositeElement__Basic_11]), - ) - states[CompositeElement_Cardinality_Asterisk].AppendTransitions( - parser.NewAtomTransition(states[CompositeElement__Basic_7], Keyword_Asterisk, nil), + states[CompositeElement__BlockEnd].AppendTransitions( + parser.NewEpsilonTransition(states[CompositeElement__Basic_9]), ) states[CompositeElement__Basic_7].AppendTransitions( - parser.NewEpsilonTransition(states[CompositeElement__BlockEnd_1]), - ) - states[CompositeElement_Cardinality_Plus].AppendTransitions( - parser.NewAtomTransition(states[CompositeElement__Basic_8], Keyword_Plus, nil), + parser.NewAtomTransition(states[CompositeElement__Basic_8], TokenGroup_Cardinality, nil), ) states[CompositeElement__Basic_8].AppendTransitions( - parser.NewEpsilonTransition(states[CompositeElement__BlockEnd_1]), - ) - states[CompositeElement_Cardinality_Question].AppendTransitions( - parser.NewAtomTransition(states[CompositeElement__Basic_9], Keyword_Question, nil), - ) - states[CompositeElement__Basic_9].AppendTransitions( - parser.NewEpsilonTransition(states[CompositeElement__BlockEnd_1]), - ) - states[CompositeElement__Basic_10].AppendTransitions( - parser.NewEpsilonTransition(states[CompositeElement_Cardinality_Asterisk]), - parser.NewEpsilonTransition(states[CompositeElement_Cardinality_Plus]), - parser.NewEpsilonTransition(states[CompositeElement_Cardinality_Question]), - ) - states[CompositeElement__BlockEnd_1].AppendTransitions( parser.NewEpsilonTransition(states[CompositeElement__Stop]), ) - states[CompositeElement__Basic_11].AppendTransitions( - parser.NewEpsilonTransition(states[CompositeElement__Basic_10]), - parser.NewEpsilonTransition(states[CompositeElement__BlockEnd_1]), + states[CompositeElement__Basic_9].AppendTransitions( + parser.NewEpsilonTransition(states[CompositeElement__Basic_7]), + parser.NewEpsilonTransition(states[CompositeElement__Basic_8]), ) - decisionStates := make([]*parser.RuntimeATNState, 39) + decisionStates := make([]*parser.RuntimeATNState, 54) decisionStates[0] = states[Grammar__Basic_1] - decisionStates[1] = states[Grammar__Basic_12] + decisionStates[1] = states[Grammar__Basic_14] decisionStates[2] = states[Grammar__LoopEntry] decisionStates[3] = states[Interface__LoopEntry_0] decisionStates[4] = states[Interface__Basic_1] @@ -1541,37 +2051,52 @@ func BuildATN() *parser.RuntimeATN { decisionStates[8] = states[ParserRule__Basic_1] decisionStates[9] = states[ParserRule__Basic_3] decisionStates[10] = states[ParserRule__Basic_6] - decisionStates[11] = states[Token__Basic_2] - decisionStates[12] = states[Token__Basic_3] - decisionStates[13] = states[Token__Basic_5] - decisionStates[14] = states[TokenGroup__Basic_4] - decisionStates[15] = states[TokenGroup__LoopEntry] - decisionStates[16] = states[Alternatives__LoopBack] - decisionStates[17] = states[Alternatives__Basic_3] - decisionStates[18] = states[Group__LoopBack] - decisionStates[19] = states[Group__Basic_3] - decisionStates[20] = states[Element__Basic_10] - decisionStates[21] = states[Element__Basic_14] - decisionStates[22] = states[Element__Basic_15] - decisionStates[23] = states[Assignment__Basic_3] - decisionStates[24] = states[Assignable__Basic_8] - decisionStates[25] = states[AssignableWithoutAlts__Basic_6] - decisionStates[26] = states[AssignableAlternatives__LoopBack] - decisionStates[27] = states[AssignableAlternatives__Basic_3] - decisionStates[28] = states[CrossRef__Basic_2] - decisionStates[29] = states[Action__Basic_2] - decisionStates[30] = states[Action__Basic_4] - decisionStates[31] = states[CompositeRule__Basic_2] - decisionStates[32] = states[CompositeAlternatives__LoopBack] - decisionStates[33] = states[CompositeAlternatives__Basic_3] - decisionStates[34] = states[CompositeGroup__LoopBack] - decisionStates[35] = states[CompositeGroup__Basic_3] - decisionStates[36] = states[CompositeElement__Basic_6] - decisionStates[37] = states[CompositeElement__Basic_10] - decisionStates[38] = states[CompositeElement__Basic_11] - decisionMap := make([]*parser.RuntimeATNState, 39) + decisionStates[11] = states[TokenDecl__Basic_2] + decisionStates[12] = states[TokenDecl__Basic_6] + decisionStates[13] = states[TokenDecl__Basic_8] + decisionStates[14] = states[TokenContent__Basic_4] + decisionStates[15] = states[TokenCommand__Basic_3] + decisionStates[16] = states[TokenCommand__Basic_6] + decisionStates[17] = states[TokenCommand__Basic_8] + decisionStates[18] = states[TokenGroup__Basic_2] + decisionStates[19] = states[TokenGroup__Basic_7] + decisionStates[20] = states[TokenGroup__LoopEntry] + decisionStates[21] = states[TokenGroup__Basic_10] + decisionStates[22] = states[TokenGroup__Basic_12] + decisionStates[23] = states[TokenMode__Basic_2] + decisionStates[24] = states[TokenMode__LoopEntry] + decisionStates[25] = states[TokenModeMember__Basic_10] + decisionStates[26] = states[TokenUsage__Basic_2] + decisionStates[27] = states[TokenUsage__Basic_5] + decisionStates[28] = states[TokenUsage__Basic_7] + decisionStates[29] = states[KeywordUsage__Basic_2] + decisionStates[30] = states[KeywordUsage__Basic_6] + decisionStates[31] = states[KeywordUsage__Basic_8] + decisionStates[32] = states[KeywordSelector__Basic_1] + decisionStates[33] = states[Alternatives__LoopBack] + decisionStates[34] = states[Alternatives__Basic_3] + decisionStates[35] = states[Group__LoopBack] + decisionStates[36] = states[Group__Basic_3] + decisionStates[37] = states[Element__Basic_10] + decisionStates[38] = states[Element__Basic_13] + decisionStates[39] = states[Assignment__Basic_3] + decisionStates[40] = states[Assignable__Basic_8] + decisionStates[41] = states[AssignableWithoutAlts__Basic_6] + decisionStates[42] = states[AssignableAlternatives__LoopBack] + decisionStates[43] = states[AssignableAlternatives__Basic_3] + decisionStates[44] = states[CrossRef__Basic_2] + decisionStates[45] = states[Action__Basic_2] + decisionStates[46] = states[Action__Basic_4] + decisionStates[47] = states[CompositeRule__Basic_2] + decisionStates[48] = states[CompositeAlternatives__LoopBack] + decisionStates[49] = states[CompositeAlternatives__Basic_3] + decisionStates[50] = states[CompositeGroup__LoopBack] + decisionStates[51] = states[CompositeGroup__Basic_3] + decisionStates[52] = states[CompositeElement__Basic_6] + decisionStates[53] = states[CompositeElement__Basic_9] + decisionMap := make([]*parser.RuntimeATNState, 54) decisionMap[0] = states[Grammar__Basic_1] - decisionMap[1] = states[Grammar__Basic_12] + decisionMap[1] = states[Grammar__Basic_14] decisionMap[2] = states[Grammar__LoopEntry] decisionMap[3] = states[Interface__LoopEntry_0] decisionMap[4] = states[Interface__Basic_1] @@ -1581,33 +2106,48 @@ func BuildATN() *parser.RuntimeATN { decisionMap[8] = states[ParserRule__Basic_1] decisionMap[9] = states[ParserRule__Basic_3] decisionMap[10] = states[ParserRule__Basic_6] - decisionMap[11] = states[Token__Basic_2] - decisionMap[12] = states[Token__Basic_3] - decisionMap[13] = states[Token__Basic_5] - decisionMap[14] = states[TokenGroup__Basic_4] - decisionMap[15] = states[TokenGroup__LoopEntry] - decisionMap[16] = states[Alternatives__LoopBack] - decisionMap[17] = states[Alternatives__Basic_3] - decisionMap[18] = states[Group__LoopBack] - decisionMap[19] = states[Group__Basic_3] - decisionMap[20] = states[Element__Basic_10] - decisionMap[21] = states[Element__Basic_14] - decisionMap[22] = states[Element__Basic_15] - decisionMap[23] = states[Assignment__Basic_3] - decisionMap[24] = states[Assignable__Basic_8] - decisionMap[25] = states[AssignableWithoutAlts__Basic_6] - decisionMap[26] = states[AssignableAlternatives__LoopBack] - decisionMap[27] = states[AssignableAlternatives__Basic_3] - decisionMap[28] = states[CrossRef__Basic_2] - decisionMap[29] = states[Action__Basic_2] - decisionMap[30] = states[Action__Basic_4] - decisionMap[31] = states[CompositeRule__Basic_2] - decisionMap[32] = states[CompositeAlternatives__LoopBack] - decisionMap[33] = states[CompositeAlternatives__Basic_3] - decisionMap[34] = states[CompositeGroup__LoopBack] - decisionMap[35] = states[CompositeGroup__Basic_3] - decisionMap[36] = states[CompositeElement__Basic_6] - decisionMap[37] = states[CompositeElement__Basic_10] - decisionMap[38] = states[CompositeElement__Basic_11] + decisionMap[11] = states[TokenDecl__Basic_2] + decisionMap[12] = states[TokenDecl__Basic_6] + decisionMap[13] = states[TokenDecl__Basic_8] + decisionMap[14] = states[TokenContent__Basic_4] + decisionMap[15] = states[TokenCommand__Basic_3] + decisionMap[16] = states[TokenCommand__Basic_6] + decisionMap[17] = states[TokenCommand__Basic_8] + decisionMap[18] = states[TokenGroup__Basic_2] + decisionMap[19] = states[TokenGroup__Basic_7] + decisionMap[20] = states[TokenGroup__LoopEntry] + decisionMap[21] = states[TokenGroup__Basic_10] + decisionMap[22] = states[TokenGroup__Basic_12] + decisionMap[23] = states[TokenMode__Basic_2] + decisionMap[24] = states[TokenMode__LoopEntry] + decisionMap[25] = states[TokenModeMember__Basic_10] + decisionMap[26] = states[TokenUsage__Basic_2] + decisionMap[27] = states[TokenUsage__Basic_5] + decisionMap[28] = states[TokenUsage__Basic_7] + decisionMap[29] = states[KeywordUsage__Basic_2] + decisionMap[30] = states[KeywordUsage__Basic_6] + decisionMap[31] = states[KeywordUsage__Basic_8] + decisionMap[32] = states[KeywordSelector__Basic_1] + decisionMap[33] = states[Alternatives__LoopBack] + decisionMap[34] = states[Alternatives__Basic_3] + decisionMap[35] = states[Group__LoopBack] + decisionMap[36] = states[Group__Basic_3] + decisionMap[37] = states[Element__Basic_10] + decisionMap[38] = states[Element__Basic_13] + decisionMap[39] = states[Assignment__Basic_3] + decisionMap[40] = states[Assignable__Basic_8] + decisionMap[41] = states[AssignableWithoutAlts__Basic_6] + decisionMap[42] = states[AssignableAlternatives__LoopBack] + decisionMap[43] = states[AssignableAlternatives__Basic_3] + decisionMap[44] = states[CrossRef__Basic_2] + decisionMap[45] = states[Action__Basic_2] + decisionMap[46] = states[Action__Basic_4] + decisionMap[47] = states[CompositeRule__Basic_2] + decisionMap[48] = states[CompositeAlternatives__LoopBack] + decisionMap[49] = states[CompositeAlternatives__Basic_3] + decisionMap[50] = states[CompositeGroup__LoopBack] + decisionMap[51] = states[CompositeGroup__Basic_3] + decisionMap[52] = states[CompositeElement__Basic_6] + decisionMap[53] = states[CompositeElement__Basic_9] return parser.NewRuntimeATN(states, decisionStates, decisionMap) } diff --git a/internal/grammar/completion_gen.go b/internal/grammar/completion_gen.go index 52fd0b95..cf12aa91 100644 --- a/internal/grammar/completion_gen.go +++ b/internal/grammar/completion_gen.go @@ -18,7 +18,9 @@ type FastbeltCompletionFilter interface { FilterReferenceTypeType(ctx context.Context, reference *core.Reference[Interface], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] FilterSimpleTypeType(ctx context.Context, reference *core.Reference[Interface], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] FilterParserRuleReturnType(ctx context.Context, reference *core.Reference[Interface], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] + FilterTokenCommandMode(ctx context.Context, reference *core.Reference[TokenMode], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] FilterTokenGroupTokenRefs(ctx context.Context, reference *core.Reference[AbstractTokenRule], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] + FilterTokenUsageTokenRef(ctx context.Context, reference *core.Reference[AbstractTokenRule], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] FilterAssignmentProperty(ctx context.Context, reference *core.Reference[Field], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] FilterCrossRefType(ctx context.Context, reference *core.Reference[Interface], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] FilterRuleCallRule(ctx context.Context, reference *core.Reference[AbstractRule], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] @@ -48,10 +50,18 @@ func (*DefaultFastbeltCompletionFilter) FilterParserRuleReturnType(_ context.Con return in } +func (*DefaultFastbeltCompletionFilter) FilterTokenCommandMode(_ context.Context, _ *core.Reference[TokenMode], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] { + return in +} + func (*DefaultFastbeltCompletionFilter) FilterTokenGroupTokenRefs(_ context.Context, _ *core.Reference[AbstractTokenRule], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] { return in } +func (*DefaultFastbeltCompletionFilter) FilterTokenUsageTokenRef(_ context.Context, _ *core.Reference[AbstractTokenRule], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] { + return in +} + func (*DefaultFastbeltCompletionFilter) FilterAssignmentProperty(_ context.Context, _ *core.Reference[Field], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] { return in } @@ -127,6 +137,18 @@ var FastbeltCompletionDispatch = map[string]FastbeltCompletionDispatchFunc{ candidates := scopes.ScopeParserRuleReturnType(ctx, ref).AllElements() return filter.FilterParserRuleReturnType(ctx, ref, candidates) }, + "TokenCommand.Mode": func(ctx context.Context, sc *service.Container, owner core.AstNode) iter.Seq[*core.SymbolDescription] { + typedOwner, ok := owner.(TokenCommand) + if !ok { + return func(yield func(*core.SymbolDescription) bool) {} + } + refs := service.MustGet[FastbeltReferencesConstructor](sc) + scopes := service.MustGet[FastbeltScopeProvider](sc) + filter := service.MustGet[FastbeltCompletionFilter](sc) + ref := refs.TokenCommandMode(typedOwner, nil) + candidates := scopes.ScopeTokenCommandMode(ctx, ref).AllElements() + return filter.FilterTokenCommandMode(ctx, ref, candidates) + }, "TokenGroup.TokenRefs": func(ctx context.Context, sc *service.Container, owner core.AstNode) iter.Seq[*core.SymbolDescription] { typedOwner, ok := owner.(TokenGroup) if !ok { @@ -139,6 +161,18 @@ var FastbeltCompletionDispatch = map[string]FastbeltCompletionDispatchFunc{ candidates := scopes.ScopeTokenGroupTokenRefs(ctx, ref).AllElements() return filter.FilterTokenGroupTokenRefs(ctx, ref, candidates) }, + "TokenUsage.TokenRef": func(ctx context.Context, sc *service.Container, owner core.AstNode) iter.Seq[*core.SymbolDescription] { + typedOwner, ok := owner.(TokenUsage) + if !ok { + return func(yield func(*core.SymbolDescription) bool) {} + } + refs := service.MustGet[FastbeltReferencesConstructor](sc) + scopes := service.MustGet[FastbeltScopeProvider](sc) + filter := service.MustGet[FastbeltCompletionFilter](sc) + ref := refs.TokenUsageTokenRef(typedOwner, nil) + candidates := scopes.ScopeTokenUsageTokenRef(ctx, ref).AllElements() + return filter.FilterTokenUsageTokenRef(ctx, ref, candidates) + }, "Assignment.Property": func(ctx context.Context, sc *service.Container, owner core.AstNode) iter.Seq[*core.SymbolDescription] { typedOwner, ok := owner.(Assignment) if !ok { @@ -277,11 +311,21 @@ func (a *FastbeltCompletionAdapter) HasAssignment(node core.AstNode, property st case "Type": return n.Type() != nil } + case TokenCommand: + switch property { + case "Mode": + return n.Mode() != nil + } case TokenGroup: switch property { case "TokenRefs": return n.TokenRefs() != nil } + case TokenUsage: + switch property { + case "TokenRef": + return n.TokenRef() != nil + } } return false } diff --git a/internal/grammar/completion_parser_gen.go b/internal/grammar/completion_parser_gen.go index ee24399e..58080353 100644 --- a/internal/grammar/completion_parser_gen.go +++ b/internal/grammar/completion_parser_gen.go @@ -71,7 +71,7 @@ func (p *CompletionParser) ParseGrammar() { { p.cp.MarkAssignment("Terminals") p.state.EnterRule(Grammar__Basic_5) - p.ParseToken() + p.ParseTokenDecl() p.state.ExitRule() p.cp.ClearAssignment() } @@ -85,16 +85,24 @@ func (p *CompletionParser) ParseGrammar() { } case 3: { - p.cp.MarkAssignment("Interfaces") + p.cp.MarkAssignment("TokenModes") p.state.EnterRule(Grammar__Basic_9) - p.ParseInterface() + p.ParseTokenMode() p.state.ExitRule() p.cp.ClearAssignment() } case 4: { - p.cp.MarkAssignment("Composites") + p.cp.MarkAssignment("Interfaces") p.state.EnterRule(Grammar__Basic_11) + p.ParseInterface() + p.state.ExitRule() + p.cp.ClearAssignment() + } + case 5: + { + p.cp.MarkAssignment("Composites") + p.state.EnterRule(Grammar__Basic_13) p.ParseCompositeRule() p.state.ExitRule() p.cp.ClearAssignment() @@ -319,22 +327,15 @@ func (p *CompletionParser) ParseParserRule() { } } -func (p *CompletionParser) ParseToken() { - p.cp.EnterRule("Token", Token__Start) +func (p *CompletionParser) ParseTokenDecl() { + p.cp.EnterRule("TokenDecl", TokenDecl__Start) defer p.cp.ExitRule() - p.cp.RecordSnapshot(Token__Basic_3) - p.state.Sync(Token__Basic_3) - switch prediction, _ := p.lookahead.TokenAlternatives(p.state); prediction { - case 0: - { - p.cp.MarkAssignment("Type") - p.state.Consume(Keyword_hidden) - p.cp.ClearAssignment() - } - case 1: - { - p.cp.MarkAssignment("Type") - p.state.Consume(Keyword_comment) + { + p.cp.RecordSnapshot(TokenDecl__Basic_2) + p.state.Sync(TokenDecl__Basic_2) + if p.lookahead.TokenDeclModifierOptional(p.state) { + p.cp.MarkAssignment("Modifier") + p.state.Consume(TokenGroup_TokenModifier) p.cp.ClearAssignment() } } @@ -349,14 +350,116 @@ func (p *CompletionParser) ParseToken() { { p.state.Consume(Keyword_Colon) } + { + p.cp.MarkAssignment("Content") + p.state.EnterRule(TokenDecl__Basic_6) + p.ParseTokenContent() + p.state.ExitRule() + p.cp.ClearAssignment() + } + { + p.cp.RecordSnapshot(TokenDecl__Basic_6) + p.state.Sync(TokenDecl__Basic_6) + if p.lookahead.TokenDeclCommandOptional(p.state) { + p.cp.MarkAssignment("Command") + p.state.EnterRule(TokenDecl__Basic_5) + p.ParseTokenCommand() + p.state.ExitRule() + p.cp.ClearAssignment() + } + } + { + if p.lookahead.TokenDeclSemicolonOptional(p.state) { + p.state.Consume(Keyword_Semicolon) + } + } +} + +func (p *CompletionParser) ParseTokenContent() { + p.cp.EnterRule("TokenContent", TokenContent__Start) + defer p.cp.ExitRule() + switch prediction, failure := p.lookahead.TokenContentAlternatives(p.state); prediction { + case 0: + { + p.state.EnterRule(TokenContent__Basic_1) + p.ParseRegexpTokenContent() + p.state.ExitRule() + } + case 1: + { + p.state.EnterRule(TokenContent__Basic_3) + p.ParseKeywordTokenContent() + p.state.ExitRule() + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } +} + +func (p *CompletionParser) ParseRegexpTokenContent() { + p.cp.EnterRule("RegexpTokenContent", RegexpTokenContent__Start) + defer p.cp.ExitRule() { p.cp.MarkAssignment("Regexp") p.state.Consume(Token_RegexLiteral) p.cp.ClearAssignment() } +} + +func (p *CompletionParser) ParseKeywordTokenContent() { + p.cp.EnterRule("KeywordTokenContent", KeywordTokenContent__Start) + defer p.cp.ExitRule() { - if p.lookahead.TokenSemicolonOptional(p.state) { - p.state.Consume(Keyword_Semicolon) + p.cp.MarkAssignment("Keyword") + p.state.EnterRule(KeywordTokenContent__Basic_1) + p.ParseKeyword() + p.state.ExitRule() + p.cp.ClearAssignment() + } +} + +func (p *CompletionParser) ParseTokenCommand() { + p.cp.EnterRule("TokenCommand", TokenCommand__Start) + defer p.cp.ExitRule() + { + p.state.Consume(Keyword_DashGreaterThan) + } + { + p.cp.MarkAssignment("Type") + switch prediction, _ := p.lookahead.TokenCommandTypeAlternatives(p.state); prediction { + case 0: + p.state.Consume(Keyword_push) + case 1: + p.state.Consume(Keyword_pop) + case 2: + p.state.Consume(Keyword_mode) + } + p.cp.ClearAssignment() + } + p.cp.RecordSnapshot(TokenCommand__Basic_8) + p.state.Sync(TokenCommand__Basic_8) + if p.lookahead.TokenCommandOptional(p.state) { + { + p.state.Consume(Keyword_LeftParen) + } + switch prediction, failure := p.lookahead.TokenCommandAlternatives(p.state); prediction { + case 0: + { + p.cp.MarkAssignment("Mode") + p.state.Consume(Token_ID) + p.cp.ClearAssignment() + } + case 1: + { + p.cp.MarkAssignment("Default") + p.state.Consume(Keyword_default) + p.cp.ClearAssignment() + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } + { + p.state.Consume(Keyword_RightParen) } } } @@ -364,6 +467,15 @@ func (p *CompletionParser) ParseToken() { func (p *CompletionParser) ParseTokenGroup() { p.cp.EnterRule("TokenGroup", TokenGroup__Start) defer p.cp.ExitRule() + { + p.cp.RecordSnapshot(TokenGroup__Basic_2) + p.state.Sync(TokenGroup__Basic_2) + if p.lookahead.TokenGroupModifierOptional(p.state) { + p.cp.MarkAssignment("Modifier") + p.state.Consume(TokenGroup_TokenModifier) + p.cp.ClearAssignment() + } + } { p.state.Consume(Keyword_token) } @@ -391,19 +503,19 @@ loop0: } case 1: { - p.state.Consume(Keyword_keywords) - } - { - p.cp.MarkAssignment("Regexps") - p.state.Consume(Token_RegexLiteral) + p.cp.MarkAssignment("Keywords") + p.state.EnterRule(TokenGroup__Basic_5) + p.ParseKeyword() + p.state.ExitRule() p.cp.ClearAssignment() } case 2: { - p.cp.MarkAssignment("Keywords") - p.state.EnterRule(TokenGroup__Basic_3) - p.ParseKeyword() - p.state.ExitRule() + p.state.Consume(Keyword_keywords) + } + { + p.cp.MarkAssignment("KeywordSelectors") + p.state.Consume(Token_RegexLiteral) p.cp.ClearAssignment() } default: @@ -415,6 +527,221 @@ loop0: { p.state.Consume(Keyword_RightBrace) } + { + p.cp.RecordSnapshot(TokenGroup__Basic_10) + p.state.Sync(TokenGroup__Basic_10) + if p.lookahead.TokenGroupCommandOptional(p.state) { + p.cp.MarkAssignment("Command") + p.state.EnterRule(TokenGroup__Basic_9) + p.ParseTokenCommand() + p.state.ExitRule() + p.cp.ClearAssignment() + } + } + { + if p.lookahead.TokenGroupSemicolonOptional(p.state) { + p.state.Consume(Keyword_Semicolon) + } + } +} + +func (p *CompletionParser) ParseTokenMode() { + p.cp.EnterRule("TokenMode", TokenMode__Start) + defer p.cp.ExitRule() + { + p.state.Consume(Keyword_token) + } + { + p.state.Consume(Keyword_mode) + } + switch prediction, failure := p.lookahead.TokenModeAlternatives(p.state); prediction { + case 0: + { + p.cp.MarkAssignment("Name") + p.state.Consume(Token_ID) + p.cp.ClearAssignment() + } + case 1: + { + p.cp.MarkAssignment("Default") + p.state.Consume(Keyword_default) + p.cp.ClearAssignment() + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } + { + p.state.Consume(Keyword_LeftBrace) + } + { + p.cp.RecordSnapshot(TokenMode__LoopEntry) + p.state.Sync(TokenMode__LoopEntry) + for p.lookahead.TokenModeMembersLoop(p.state) { + p.cp.MarkAssignment("Members") + p.state.EnterRule(TokenMode__Basic_4) + p.ParseTokenModeMember() + p.state.ExitRule() + p.cp.ClearAssignment() + p.cp.RecordSnapshot(TokenMode__LoopEntry) + p.state.Sync(TokenMode__LoopEntry) + } + } + { + p.state.Consume(Keyword_RightBrace) + } +} + +func (p *CompletionParser) ParseTokenModeMember() { + p.cp.EnterRule("TokenModeMember", TokenModeMember__Start) + defer p.cp.ExitRule() + switch prediction, failure := p.lookahead.TokenModeMemberAlternatives(p.state); prediction { + case 0: + { + p.state.EnterRule(TokenModeMember__Basic_1) + p.ParseTokenDeclUsage() + p.state.ExitRule() + } + case 1: + { + p.state.EnterRule(TokenModeMember__Basic_3) + p.ParseTokenGroupUsage() + p.state.ExitRule() + } + case 2: + { + p.state.EnterRule(TokenModeMember__Basic_5) + p.ParseTokenUsage() + p.state.ExitRule() + } + case 3: + { + p.state.EnterRule(TokenModeMember__Basic_7) + p.ParseKeywordUsage() + p.state.ExitRule() + } + case 4: + { + p.state.EnterRule(TokenModeMember__Basic_9) + p.ParseKeywordSelector() + p.state.ExitRule() + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } +} + +func (p *CompletionParser) ParseTokenDeclUsage() { + p.cp.EnterRule("TokenDeclUsage", TokenDeclUsage__Start) + defer p.cp.ExitRule() + { + p.cp.MarkAssignment("Declaration") + p.state.EnterRule(TokenDeclUsage__Basic_1) + p.ParseTokenDecl() + p.state.ExitRule() + p.cp.ClearAssignment() + } +} + +func (p *CompletionParser) ParseTokenGroupUsage() { + p.cp.EnterRule("TokenGroupUsage", TokenGroupUsage__Start) + defer p.cp.ExitRule() + { + p.cp.MarkAssignment("Group") + p.state.EnterRule(TokenGroupUsage__Basic_1) + p.ParseTokenGroup() + p.state.ExitRule() + p.cp.ClearAssignment() + } +} + +func (p *CompletionParser) ParseTokenUsage() { + p.cp.EnterRule("TokenUsage", TokenUsage__Start) + defer p.cp.ExitRule() + { + p.cp.RecordSnapshot(TokenUsage__Basic_2) + p.state.Sync(TokenUsage__Basic_2) + if p.lookahead.TokenUsageModifierOptional(p.state) { + p.cp.MarkAssignment("Modifier") + p.state.Consume(TokenGroup_TokenModifier) + p.cp.ClearAssignment() + } + } + { + p.cp.MarkAssignment("TokenRef") + p.state.Consume(Token_ID) + p.cp.ClearAssignment() + } + { + p.cp.RecordSnapshot(TokenUsage__Basic_5) + p.state.Sync(TokenUsage__Basic_5) + if p.lookahead.TokenUsageCommandOptional(p.state) { + p.cp.MarkAssignment("Command") + p.state.EnterRule(TokenUsage__Basic_4) + p.ParseTokenCommand() + p.state.ExitRule() + p.cp.ClearAssignment() + } + } + { + if p.lookahead.TokenUsageSemicolonOptional(p.state) { + p.state.Consume(Keyword_Semicolon) + } + } +} + +func (p *CompletionParser) ParseKeywordUsage() { + p.cp.EnterRule("KeywordUsage", KeywordUsage__Start) + defer p.cp.ExitRule() + { + p.cp.RecordSnapshot(KeywordUsage__Basic_2) + p.state.Sync(KeywordUsage__Basic_2) + if p.lookahead.KeywordUsageModifierOptional(p.state) { + p.cp.MarkAssignment("Modifier") + p.state.Consume(TokenGroup_TokenModifier) + p.cp.ClearAssignment() + } + } + { + p.cp.MarkAssignment("Keyword") + p.state.EnterRule(KeywordUsage__Basic_6) + p.ParseKeyword() + p.state.ExitRule() + p.cp.ClearAssignment() + } + { + p.cp.RecordSnapshot(KeywordUsage__Basic_6) + p.state.Sync(KeywordUsage__Basic_6) + if p.lookahead.KeywordUsageCommandOptional(p.state) { + p.cp.MarkAssignment("Command") + p.state.EnterRule(KeywordUsage__Basic_5) + p.ParseTokenCommand() + p.state.ExitRule() + p.cp.ClearAssignment() + } + } + { + if p.lookahead.KeywordUsageSemicolonOptional(p.state) { + p.state.Consume(Keyword_Semicolon) + } + } +} + +func (p *CompletionParser) ParseKeywordSelector() { + p.cp.EnterRule("KeywordSelector", KeywordSelector__Start) + defer p.cp.ExitRule() + { + p.state.Consume(Keyword_keywords) + } + { + p.cp.MarkAssignment("Selector") + p.state.Consume(Token_RegexLiteral) + p.cp.ClearAssignment() + } + { + if p.lookahead.KeywordSelectorSemicolonOptional(p.state) { + p.state.Consume(Keyword_Semicolon) + } + } } func (p *CompletionParser) ParseAlternatives() { @@ -514,20 +841,11 @@ func (p *CompletionParser) ParseElement() { p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) } { - p.cp.RecordSnapshot(Element__Basic_15) - p.state.Sync(Element__Basic_15) - switch prediction, _ := p.lookahead.ElementCardinalityAlternatives(p.state); prediction { - case 0: - p.cp.MarkAssignment("Cardinality") - p.state.Consume(Keyword_Asterisk) - p.cp.ClearAssignment() - case 1: - p.cp.MarkAssignment("Cardinality") - p.state.Consume(Keyword_Plus) - p.cp.ClearAssignment() - case 2: + p.cp.RecordSnapshot(Element__Basic_13) + p.state.Sync(Element__Basic_13) + if p.lookahead.ElementCardinalityOptional(p.state) { p.cp.MarkAssignment("Cardinality") - p.state.Consume(Keyword_Question) + p.state.Consume(TokenGroup_Cardinality) p.cp.ClearAssignment() } } @@ -860,20 +1178,11 @@ func (p *CompletionParser) ParseCompositeElement() { p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) } { - p.cp.RecordSnapshot(CompositeElement__Basic_11) - p.state.Sync(CompositeElement__Basic_11) - switch prediction, _ := p.lookahead.CompositeElementCardinalityAlternatives(p.state); prediction { - case 0: - p.cp.MarkAssignment("Cardinality") - p.state.Consume(Keyword_Asterisk) - p.cp.ClearAssignment() - case 1: - p.cp.MarkAssignment("Cardinality") - p.state.Consume(Keyword_Plus) - p.cp.ClearAssignment() - case 2: + p.cp.RecordSnapshot(CompositeElement__Basic_9) + p.state.Sync(CompositeElement__Basic_9) + if p.lookahead.CompositeElementCardinalityOptional(p.state) { p.cp.MarkAssignment("Cardinality") - p.state.Consume(Keyword_Question) + p.state.Consume(TokenGroup_Cardinality) p.cp.ClearAssignment() } } diff --git a/internal/grammar/grammar.fb b/internal/grammar/grammar.fb index 478d9f5b..e2cae2d1 100644 --- a/internal/grammar/grammar.fb +++ b/internal/grammar/grammar.fb @@ -4,16 +4,18 @@ interface Grammar { Name string Rules []ParserRule Composites []CompositeRule - Terminals []Token + Terminals []TokenDecl TokenGroups []TokenGroup + TokenModes []TokenMode Interfaces []Interface } entry Grammar: "grammar" Name=ID ";"? ( Rules+=ParserRule - | Terminals+=Token + | Terminals+=TokenDecl | TokenGroups+=TokenGroup + | TokenModes+=TokenMode | Interfaces+=Interface | Composites+=CompositeRule )* @@ -72,8 +74,11 @@ interface AbstractRuleWithBody extends AbstractRule { Body Element } -// Base type for Token and TokenGroup -interface AbstractTokenRule extends AbstractRule{} +// Base type for TokenDecl and TokenGroup +interface AbstractTokenRule extends AbstractRule { + Modifier string + Command TokenCommand +} interface ParserRule extends AbstractRuleWithBody { Entry bool @@ -82,23 +87,98 @@ interface ParserRule extends AbstractRuleWithBody { ParserRule: (Entry?="entry")? Name=ID ("returns" ReturnType=[Interface:ID])? ":" Body=Alternatives ";"? -interface Token extends AbstractTokenRule { - Type string +interface TokenDecl extends AbstractTokenRule { + Content TokenContent +} + +TokenDecl: Modifier=TokenModifier? "token" Name=ID ":" Content=TokenContent Command=TokenCommand? ";"? + +interface TokenContent {} + +TokenContent: RegexpTokenContent | KeywordTokenContent + +interface RegexpTokenContent extends TokenContent { Regexp string } -Token: (Type="hidden" | Type="comment")? "token" Name=ID ":" Regexp=RegexLiteral ";"? +RegexpTokenContent: Regexp=RegexLiteral + +interface KeywordTokenContent extends TokenContent { + Keyword Keyword +} + +KeywordTokenContent: Keyword=Keyword + +interface TokenCommand { + Type string + Mode *TokenMode + Default bool +} + +TokenCommand: "->" Type=("push"|"pop"|"mode") ("(" (Mode=[TokenMode:ID] | Default?="default") ")")?; interface TokenGroup extends AbstractTokenRule { - TokenRefs []*AbstractTokenRule - Regexps []string - Keywords []Keyword + TokenRefs []*AbstractTokenRule + Keywords []Keyword + KeywordSelectors []string } -TokenGroup: "token" "group" Name=ID "{" - (TokenRefs+=[AbstractTokenRule:ID] | "keywords" Regexps+=RegexLiteral | Keywords+=Keyword)* +TokenGroup: Modifier=TokenModifier? "token" "group" Name=ID "{" + ( + TokenRefs+=[AbstractTokenRule:ID] + | Keywords+=Keyword + | "keywords" KeywordSelectors+=RegexLiteral + )* +"}" Command=TokenCommand? ";"? + +interface TokenMode { + Name string + Default bool + Members []TokenModeMember +} + +TokenMode: "token" "mode" (Name=ID | Default?="default") "{" + Members+=TokenModeMember* "}" +interface TokenModeMember {} + +TokenModeMember: TokenDeclUsage | TokenGroupUsage | TokenUsage | KeywordUsage | KeywordSelector + +interface TokenDeclUsage extends TokenModeMember { + Declaration TokenDecl +} + +TokenDeclUsage: Declaration=TokenDecl + +interface TokenGroupUsage extends TokenModeMember { + Group TokenGroup +} + +TokenGroupUsage: Group=TokenGroup + +interface TokenUsage extends TokenModeMember { + Modifier string + TokenRef *AbstractTokenRule + Command TokenCommand +} + +TokenUsage: Modifier=TokenModifier? TokenRef=[AbstractTokenRule:ID] Command=TokenCommand? ";"? + +interface KeywordUsage extends TokenModeMember { + Modifier string + Keyword Keyword + Command TokenCommand +} + +KeywordUsage: Modifier=TokenModifier? Keyword=Keyword Command=TokenCommand? ";"? + +interface KeywordSelector extends TokenModeMember { + Selector string +} + +KeywordSelector: "keywords" Selector=RegexLiteral ";"? + interface Element { Cardinality string } @@ -115,7 +195,7 @@ interface Group extends Element { Group returns Element: Element ({Group.Elements+=current} Elements+=Element+)? -Element: (Keyword | Assignment | RuleCall | Action | "(" Alternatives ")") Cardinality=("*" | "+" | "?")? +Element: (Keyword | Assignment | RuleCall | Action | "(" Alternatives ")") Cardinality=Cardinality? interface Keyword extends Assignable { Value string @@ -169,7 +249,18 @@ CompositeAlternatives returns Element: CompositeGroup ({Alternatives.Alts+=curre CompositeGroup returns Element: CompositeElement ({Group.Elements+=current} Elements+=CompositeElement+)? -CompositeElement returns Element: (Keyword | RuleCall | "(" CompositeAlternatives ")") Cardinality=("*" | "+" | "?")? +CompositeElement returns Element: (Keyword | RuleCall | "(" CompositeAlternatives ")") Cardinality=Cardinality? + +token group Cardinality { + "*" + "+" + "?" +} + +token group TokenModifier { + "hidden" + "comment" +} // Move comments in front of RegexLiteral comment token SL_COMMENT: /\/\/[^\r\n]*/ @@ -178,4 +269,4 @@ comment token ML_COMMENT: /\/\*[\s\S]*?\*\// token StringLiteral: /"[^"]*"/ token ID: /[a-zA-Z_][a-zA-Z0-9_]*/ token RegexLiteral: /\/([^\r\n\[\/\\]|\\.|\[([^\r\n\]\\]|\\.)*\])+\// -hidden token WS: /[ \n\r\t]+/ +hidden token WS: /[ \n\r\t]+/ \ No newline at end of file diff --git a/internal/grammar/keyword_name.go b/internal/grammar/keyword_name.go index 584e1228..4940dc02 100644 --- a/internal/grammar/keyword_name.go +++ b/internal/grammar/keyword_name.go @@ -6,6 +6,11 @@ package grammar import "strings" +// Strip surrounding slashes from the regex literal +func RegexpValue(tokenImage string) string { + return tokenImage[1 : len(tokenImage)-1] +} + // KeywordValue returns the string value of k stripped of its surrounding quotes. func KeywordValue(k Keyword) string { return k.Value()[1 : len(k.Value())-1] diff --git a/internal/grammar/lexer_gen.go b/internal/grammar/lexer_gen.go index 81188c3f..3c479d93 100644 --- a/internal/grammar/lexer_gen.go +++ b/internal/grammar/lexer_gen.go @@ -15,10 +15,7 @@ var Keyword_LeftParen = core.NewTokenType( Keyword_LeftParen_Idx, "(", "(", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "(") { return 1 @@ -34,10 +31,7 @@ var Keyword_RightParen = core.NewTokenType( Keyword_RightParen_Idx, ")", ")", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ")") { return 1 @@ -53,10 +47,7 @@ var Keyword_Asterisk = core.NewTokenType( Keyword_Asterisk_Idx, "*", "*", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "*") { return 1 @@ -72,10 +63,7 @@ var Keyword_Plus = core.NewTokenType( Keyword_Plus_Idx, "+", "+", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "+") { return 1 @@ -91,10 +79,7 @@ var Keyword_PlusEquals = core.NewTokenType( Keyword_PlusEquals_Idx, "+=", "+=", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "+=") { return 2 @@ -110,10 +95,7 @@ var Keyword_Comma = core.NewTokenType( Keyword_Comma_Idx, ",", ",", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ",") { return 1 @@ -123,16 +105,29 @@ var Keyword_Comma = core.NewTokenType( []rune{','}, ) -const Keyword_Dot_Idx = 7 +const Keyword_DashGreaterThan_Idx = 7 + +var Keyword_DashGreaterThan = core.NewTokenType( + Keyword_DashGreaterThan_Idx, + "->", + "->", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "->") { + return 2 + } + return 0 + }, + []rune{'-'}, +) + +const Keyword_Dot_Idx = 8 var Keyword_Dot = core.NewTokenType( Keyword_Dot_Idx, ".", ".", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ".") { return 1 @@ -142,16 +137,13 @@ var Keyword_Dot = core.NewTokenType( []rune{'.'}, ) -const Keyword_Colon_Idx = 8 +const Keyword_Colon_Idx = 9 var Keyword_Colon = core.NewTokenType( Keyword_Colon_Idx, ":", ":", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ":") { return 1 @@ -161,16 +153,13 @@ var Keyword_Colon = core.NewTokenType( []rune{':'}, ) -const Keyword_Semicolon_Idx = 9 +const Keyword_Semicolon_Idx = 10 var Keyword_Semicolon = core.NewTokenType( Keyword_Semicolon_Idx, ";", ";", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ";") { return 1 @@ -180,16 +169,13 @@ var Keyword_Semicolon = core.NewTokenType( []rune{';'}, ) -const Keyword_Equals_Idx = 10 +const Keyword_Equals_Idx = 11 var Keyword_Equals = core.NewTokenType( Keyword_Equals_Idx, "=", "=", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "=") { return 1 @@ -199,16 +185,13 @@ var Keyword_Equals = core.NewTokenType( []rune{'='}, ) -const Keyword_Question_Idx = 11 +const Keyword_Question_Idx = 12 var Keyword_Question = core.NewTokenType( Keyword_Question_Idx, "?", "?", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "?") { return 1 @@ -218,16 +201,13 @@ var Keyword_Question = core.NewTokenType( []rune{'?'}, ) -const Keyword_QuestionEquals_Idx = 12 +const Keyword_QuestionEquals_Idx = 13 var Keyword_QuestionEquals = core.NewTokenType( Keyword_QuestionEquals_Idx, "?=", "?=", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "?=") { return 2 @@ -237,16 +217,13 @@ var Keyword_QuestionEquals = core.NewTokenType( []rune{'?'}, ) -const Keyword_LeftBracket_Idx = 13 +const Keyword_LeftBracket_Idx = 14 var Keyword_LeftBracket = core.NewTokenType( Keyword_LeftBracket_Idx, "[", "[", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "[") { return 1 @@ -256,16 +233,13 @@ var Keyword_LeftBracket = core.NewTokenType( []rune{'['}, ) -const Keyword_RightBracket_Idx = 14 +const Keyword_RightBracket_Idx = 15 var Keyword_RightBracket = core.NewTokenType( Keyword_RightBracket_Idx, "]", "]", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "]") { return 1 @@ -275,16 +249,13 @@ var Keyword_RightBracket = core.NewTokenType( []rune{']'}, ) -const Keyword_bool_Idx = 15 +const Keyword_bool_Idx = 16 var Keyword_bool = core.NewTokenType( Keyword_bool_Idx, "bool", "bool", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "bool") { return 4 @@ -294,16 +265,13 @@ var Keyword_bool = core.NewTokenType( []rune{'b'}, ) -const Keyword_comment_Idx = 16 +const Keyword_comment_Idx = 17 var Keyword_comment = core.NewTokenType( Keyword_comment_Idx, "comment", "comment", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "comment") { return 7 @@ -313,16 +281,13 @@ var Keyword_comment = core.NewTokenType( []rune{'c'}, ) -const Keyword_composite_Idx = 17 +const Keyword_composite_Idx = 18 var Keyword_composite = core.NewTokenType( Keyword_composite_Idx, "composite", "composite", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "composite") { return 9 @@ -332,16 +297,13 @@ var Keyword_composite = core.NewTokenType( []rune{'c'}, ) -const Keyword_current_Idx = 18 +const Keyword_current_Idx = 19 var Keyword_current = core.NewTokenType( Keyword_current_Idx, "current", "current", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "current") { return 7 @@ -351,16 +313,29 @@ var Keyword_current = core.NewTokenType( []rune{'c'}, ) -const Keyword_entry_Idx = 19 +const Keyword_default_Idx = 20 + +var Keyword_default = core.NewTokenType( + Keyword_default_Idx, + "default", + "default", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "default") { + return 7 + } + return 0 + }, + []rune{'d'}, +) + +const Keyword_entry_Idx = 21 var Keyword_entry = core.NewTokenType( Keyword_entry_Idx, "entry", "entry", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "entry") { return 5 @@ -370,16 +345,13 @@ var Keyword_entry = core.NewTokenType( []rune{'e'}, ) -const Keyword_extends_Idx = 20 +const Keyword_extends_Idx = 22 var Keyword_extends = core.NewTokenType( Keyword_extends_Idx, "extends", "extends", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "extends") { return 7 @@ -389,16 +361,13 @@ var Keyword_extends = core.NewTokenType( []rune{'e'}, ) -const Keyword_grammar_Idx = 21 +const Keyword_grammar_Idx = 23 var Keyword_grammar = core.NewTokenType( Keyword_grammar_Idx, "grammar", "grammar", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "grammar") { return 7 @@ -408,16 +377,13 @@ var Keyword_grammar = core.NewTokenType( []rune{'g'}, ) -const Keyword_group_Idx = 22 +const Keyword_group_Idx = 24 var Keyword_group = core.NewTokenType( Keyword_group_Idx, "group", "group", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "group") { return 5 @@ -427,16 +393,13 @@ var Keyword_group = core.NewTokenType( []rune{'g'}, ) -const Keyword_hidden_Idx = 23 +const Keyword_hidden_Idx = 25 var Keyword_hidden = core.NewTokenType( Keyword_hidden_Idx, "hidden", "hidden", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "hidden") { return 6 @@ -446,16 +409,13 @@ var Keyword_hidden = core.NewTokenType( []rune{'h'}, ) -const Keyword_interface_Idx = 24 +const Keyword_interface_Idx = 26 var Keyword_interface = core.NewTokenType( Keyword_interface_Idx, "interface", "interface", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "interface") { return 9 @@ -465,16 +425,13 @@ var Keyword_interface = core.NewTokenType( []rune{'i'}, ) -const Keyword_keywords_Idx = 25 +const Keyword_keywords_Idx = 27 var Keyword_keywords = core.NewTokenType( Keyword_keywords_Idx, "keywords", "keywords", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "keywords") { return 8 @@ -484,16 +441,61 @@ var Keyword_keywords = core.NewTokenType( []rune{'k'}, ) -const Keyword_returns_Idx = 26 +const Keyword_mode_Idx = 28 + +var Keyword_mode = core.NewTokenType( + Keyword_mode_Idx, + "mode", + "mode", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "mode") { + return 4 + } + return 0 + }, + []rune{'m'}, +) + +const Keyword_pop_Idx = 29 + +var Keyword_pop = core.NewTokenType( + Keyword_pop_Idx, + "pop", + "pop", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "pop") { + return 3 + } + return 0 + }, + []rune{'p'}, +) + +const Keyword_push_Idx = 30 + +var Keyword_push = core.NewTokenType( + Keyword_push_Idx, + "push", + "push", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "push") { + return 4 + } + return 0 + }, + []rune{'p'}, +) + +const Keyword_returns_Idx = 31 var Keyword_returns = core.NewTokenType( Keyword_returns_Idx, "returns", "returns", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "returns") { return 7 @@ -503,16 +505,13 @@ var Keyword_returns = core.NewTokenType( []rune{'r'}, ) -const Keyword_string_Idx = 27 +const Keyword_string_Idx = 32 var Keyword_string = core.NewTokenType( Keyword_string_Idx, "string", "string", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "string") { return 6 @@ -522,16 +521,13 @@ var Keyword_string = core.NewTokenType( []rune{'s'}, ) -const Keyword_token_Idx = 28 +const Keyword_token_Idx = 33 var Keyword_token = core.NewTokenType( Keyword_token_Idx, "token", "token", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "token") { return 5 @@ -541,16 +537,13 @@ var Keyword_token = core.NewTokenType( []rune{'t'}, ) -const Keyword_LeftBrace_Idx = 29 +const Keyword_LeftBrace_Idx = 34 var Keyword_LeftBrace = core.NewTokenType( Keyword_LeftBrace_Idx, "{", "{", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "{") { return 1 @@ -560,16 +553,13 @@ var Keyword_LeftBrace = core.NewTokenType( []rune{'{'}, ) -const Keyword_Pipe_Idx = 30 +const Keyword_Pipe_Idx = 35 var Keyword_Pipe = core.NewTokenType( Keyword_Pipe_Idx, "|", "|", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "|") { return 1 @@ -579,16 +569,13 @@ var Keyword_Pipe = core.NewTokenType( []rune{'|'}, ) -const Keyword_RightBrace_Idx = 31 +const Keyword_RightBrace_Idx = 36 var Keyword_RightBrace = core.NewTokenType( Keyword_RightBrace_Idx, "}", "}", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "}") { return 1 @@ -598,16 +585,13 @@ var Keyword_RightBrace = core.NewTokenType( []rune{'}'}, ) -const Token_SL_COMMENT_Idx = 32 +const Token_SL_COMMENT_Idx = 37 var Token_SL_COMMENT = core.NewTokenType( Token_SL_COMMENT_Idx, "SL_COMMENT", "SL_COMMENT", - core.CommentGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -689,16 +673,13 @@ var Token_SL_COMMENT_Accepting = [3]bool{ 2: true, } -const Token_ML_COMMENT_Idx = 33 +const Token_ML_COMMENT_Idx = 38 var Token_ML_COMMENT = core.NewTokenType( Token_ML_COMMENT_Idx, "ML_COMMENT", "ML_COMMENT", - core.CommentGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -814,16 +795,13 @@ var Token_ML_COMMENT_Accepting = [5]bool{ 4: true, } -const Token_StringLiteral_Idx = 34 +const Token_StringLiteral_Idx = 39 var Token_StringLiteral = core.NewTokenType( Token_StringLiteral_Idx, "StringLiteral", "StringLiteral", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -905,16 +883,13 @@ var Token_StringLiteral_Accepting = [3]bool{ 2: true, } -const Token_ID_Idx = 35 +const Token_ID_Idx = 40 var Token_ID = core.NewTokenType( Token_ID_Idx, "ID", "ID", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -979,16 +954,13 @@ var Token_ID_Accepting = [2]bool{ 1: true, } -const Token_RegexLiteral_Idx = 36 +const Token_RegexLiteral_Idx = 41 var Token_RegexLiteral = core.NewTokenType( Token_RegexLiteral_Idx, "RegexLiteral", "RegexLiteral", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -1138,16 +1110,13 @@ var Token_RegexLiteral_Accepting = [7]bool{ 5: true, } -const Token_WS_Idx = 37 +const Token_WS_Idx = 42 var Token_WS = core.NewTokenType( Token_WS_Idx, "WS", "WS", - core.SkippedGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -1212,44 +1181,80 @@ var Token_WS_Accepting = [2]bool{ 1: true, } -func NewLexer() lexer.Lexer { - return lexer.NewDefaultLexer( - Keyword_LeftParen, - Keyword_RightParen, +const TokenGroup_Cardinality_Idx = 43 + +var TokenGroup_Cardinality = core.NewTokenGroup( + TokenGroup_Cardinality_Idx, + "Cardinality", + "Cardinality", + []*core.TokenType{ Keyword_Asterisk, Keyword_Plus, - Keyword_PlusEquals, - Keyword_Comma, - Keyword_Dot, - Keyword_Colon, - Keyword_Semicolon, - Keyword_Equals, Keyword_Question, - Keyword_QuestionEquals, - Keyword_LeftBracket, - Keyword_RightBracket, - Keyword_bool, + }, +) + +const TokenGroup_TokenModifier_Idx = 44 + +var TokenGroup_TokenModifier = core.NewTokenGroup( + TokenGroup_TokenModifier_Idx, + "TokenModifier", + "TokenModifier", + []*core.TokenType{ Keyword_comment, - Keyword_composite, - Keyword_current, - Keyword_entry, - Keyword_extends, - Keyword_grammar, - Keyword_group, Keyword_hidden, - Keyword_interface, - Keyword_keywords, - Keyword_returns, - Keyword_string, - Keyword_token, - Keyword_LeftBrace, - Keyword_Pipe, - Keyword_RightBrace, - Token_SL_COMMENT, - Token_ML_COMMENT, - Token_StringLiteral, - Token_ID, - Token_RegexLiteral, - Token_WS, + }, +) + +const ( + TokenMode_default = 0 +) + +func NewLexer() lexer.Lexer { + modes := make([]*lexer.TokenMode, 1) + modes[TokenMode_default] = lexer.NewTokenMode("default", + lexer.UseTokenType(Keyword_Asterisk), + lexer.UseTokenType(Keyword_Plus), + lexer.UseTokenType(Keyword_Question), + lexer.UseTokenType(Keyword_hidden), + lexer.UseTokenType(Keyword_comment), + lexer.UseTokenType(Keyword_LeftParen), + lexer.UseTokenType(Keyword_RightParen), + lexer.UseTokenType(Keyword_PlusEquals), + lexer.UseTokenType(Keyword_Comma), + lexer.UseTokenType(Keyword_DashGreaterThan), + lexer.UseTokenType(Keyword_Dot), + lexer.UseTokenType(Keyword_Colon), + lexer.UseTokenType(Keyword_Semicolon), + lexer.UseTokenType(Keyword_Equals), + lexer.UseTokenType(Keyword_QuestionEquals), + lexer.UseTokenType(Keyword_LeftBracket), + lexer.UseTokenType(Keyword_RightBracket), + lexer.UseTokenType(Keyword_bool), + lexer.UseTokenType(Keyword_composite), + lexer.UseTokenType(Keyword_current), + lexer.UseTokenType(Keyword_default), + lexer.UseTokenType(Keyword_entry), + lexer.UseTokenType(Keyword_extends), + lexer.UseTokenType(Keyword_grammar), + lexer.UseTokenType(Keyword_group), + lexer.UseTokenType(Keyword_interface), + lexer.UseTokenType(Keyword_keywords), + lexer.UseTokenType(Keyword_mode), + lexer.UseTokenType(Keyword_pop), + lexer.UseTokenType(Keyword_push), + lexer.UseTokenType(Keyword_returns), + lexer.UseTokenType(Keyword_string), + lexer.UseTokenType(Keyword_token), + lexer.UseTokenType(Keyword_LeftBrace), + lexer.UseTokenType(Keyword_Pipe), + lexer.UseTokenType(Keyword_RightBrace), + lexer.UseTokenType(Token_SL_COMMENT).WithModifier(core.CommentModifier), + lexer.UseTokenType(Token_ML_COMMENT).WithModifier(core.CommentModifier), + lexer.UseTokenType(Token_StringLiteral), + lexer.UseTokenType(Token_ID), + lexer.UseTokenType(Token_RegexLiteral), + lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier), ) + return lexer.NewDefaultLexer(TokenMode_default, modes...) } diff --git a/internal/grammar/linker_gen.go b/internal/grammar/linker_gen.go index 08e5ad20..938bdbcc 100644 --- a/internal/grammar/linker_gen.go +++ b/internal/grammar/linker_gen.go @@ -19,7 +19,9 @@ type FastbeltScopeProvider interface { ScopeReferenceTypeType(ctx context.Context, reference *core.Reference[Interface]) core.Scope ScopeSimpleTypeType(ctx context.Context, reference *core.Reference[Interface]) core.Scope ScopeParserRuleReturnType(ctx context.Context, reference *core.Reference[Interface]) core.Scope + ScopeTokenCommandMode(ctx context.Context, reference *core.Reference[TokenMode]) core.Scope ScopeTokenGroupTokenRefs(ctx context.Context, reference *core.Reference[AbstractTokenRule]) core.Scope + ScopeTokenUsageTokenRef(ctx context.Context, reference *core.Reference[AbstractTokenRule]) core.Scope ScopeAssignmentProperty(ctx context.Context, reference *core.Reference[Field]) core.Scope ScopeCrossRefType(ctx context.Context, reference *core.Reference[Interface]) core.Scope ScopeRuleCallRule(ctx context.Context, reference *core.Reference[AbstractRule]) core.Scope @@ -51,10 +53,18 @@ func (s *DefaultFastbeltScopeProvider) ScopeParserRuleReturnType(ctx context.Con return linking.DefaultScopeOfType[Interface](reference.Owner()) } +func (s *DefaultFastbeltScopeProvider) ScopeTokenCommandMode(ctx context.Context, reference *core.Reference[TokenMode]) core.Scope { + return linking.DefaultScopeOfType[TokenMode](reference.Owner()) +} + func (s *DefaultFastbeltScopeProvider) ScopeTokenGroupTokenRefs(ctx context.Context, reference *core.Reference[AbstractTokenRule]) core.Scope { return linking.DefaultScopeOfType[AbstractTokenRule](reference.Owner()) } +func (s *DefaultFastbeltScopeProvider) ScopeTokenUsageTokenRef(ctx context.Context, reference *core.Reference[AbstractTokenRule]) core.Scope { + return linking.DefaultScopeOfType[AbstractTokenRule](reference.Owner()) +} + func (s *DefaultFastbeltScopeProvider) ScopeAssignmentProperty(ctx context.Context, reference *core.Reference[Field]) core.Scope { return linking.DefaultScopeOfType[Field](reference.Owner()) } @@ -80,7 +90,9 @@ type FastbeltReferenceLinker interface { LinkReferenceTypeType(ctx context.Context, reference *core.Reference[Interface]) (*core.SymbolDescription, *core.ReferenceError) LinkSimpleTypeType(ctx context.Context, reference *core.Reference[Interface]) (*core.SymbolDescription, *core.ReferenceError) LinkParserRuleReturnType(ctx context.Context, reference *core.Reference[Interface]) (*core.SymbolDescription, *core.ReferenceError) + LinkTokenCommandMode(ctx context.Context, reference *core.Reference[TokenMode]) (*core.SymbolDescription, *core.ReferenceError) LinkTokenGroupTokenRefs(ctx context.Context, reference *core.Reference[AbstractTokenRule]) (*core.SymbolDescription, *core.ReferenceError) + LinkTokenUsageTokenRef(ctx context.Context, reference *core.Reference[AbstractTokenRule]) (*core.SymbolDescription, *core.ReferenceError) LinkAssignmentProperty(ctx context.Context, reference *core.Reference[Field]) (*core.SymbolDescription, *core.ReferenceError) LinkCrossRefType(ctx context.Context, reference *core.Reference[Interface]) (*core.SymbolDescription, *core.ReferenceError) LinkRuleCallRule(ctx context.Context, reference *core.Reference[AbstractRule]) (*core.SymbolDescription, *core.ReferenceError) @@ -122,11 +134,21 @@ func (s *DefaultFastbeltReferenceLinker) LinkParserRuleReturnType(ctx context.Co return core.DefaultLink(scope, reference.Text()) } +func (s *DefaultFastbeltReferenceLinker) LinkTokenCommandMode(ctx context.Context, reference *core.Reference[TokenMode]) (*core.SymbolDescription, *core.ReferenceError) { + scope := s.scopeProvider().ScopeTokenCommandMode(ctx, reference) + return core.DefaultLink(scope, reference.Text()) +} + func (s *DefaultFastbeltReferenceLinker) LinkTokenGroupTokenRefs(ctx context.Context, reference *core.Reference[AbstractTokenRule]) (*core.SymbolDescription, *core.ReferenceError) { scope := s.scopeProvider().ScopeTokenGroupTokenRefs(ctx, reference) return core.DefaultLink(scope, reference.Text()) } +func (s *DefaultFastbeltReferenceLinker) LinkTokenUsageTokenRef(ctx context.Context, reference *core.Reference[AbstractTokenRule]) (*core.SymbolDescription, *core.ReferenceError) { + scope := s.scopeProvider().ScopeTokenUsageTokenRef(ctx, reference) + return core.DefaultLink(scope, reference.Text()) +} + func (s *DefaultFastbeltReferenceLinker) LinkAssignmentProperty(ctx context.Context, reference *core.Reference[Field]) (*core.SymbolDescription, *core.ReferenceError) { scope := s.scopeProvider().ScopeAssignmentProperty(ctx, reference) return core.DefaultLink(scope, reference.Text()) @@ -157,7 +179,9 @@ type FastbeltReferencesConstructor interface { ReferenceTypeType(owner core.AstNode, unit core.StringUnit) *core.Reference[Interface] SimpleTypeType(owner core.AstNode, unit core.StringUnit) *core.Reference[Interface] ParserRuleReturnType(owner core.AstNode, unit core.StringUnit) *core.Reference[Interface] + TokenCommandMode(owner core.AstNode, unit core.StringUnit) *core.Reference[TokenMode] TokenGroupTokenRefs(owner core.AstNode, unit core.StringUnit) *core.Reference[AbstractTokenRule] + TokenUsageTokenRef(owner core.AstNode, unit core.StringUnit) *core.Reference[AbstractTokenRule] AssignmentProperty(owner core.AstNode, unit core.StringUnit) *core.Reference[Field] CrossRefType(owner core.AstNode, unit core.StringUnit) *core.Reference[Interface] RuleCallRule(owner core.AstNode, unit core.StringUnit) *core.Reference[AbstractRule] @@ -199,11 +223,21 @@ func (s *DefaultFastbeltReferencesConstructor) ParserRuleReturnType(owner core.A return core.NewReference(owner, unit, fn) } +func (s *DefaultFastbeltReferencesConstructor) TokenCommandMode(owner core.AstNode, unit core.StringUnit) *core.Reference[TokenMode] { + fn := s.referenceLinker().LinkTokenCommandMode + return core.NewReference(owner, unit, fn) +} + func (s *DefaultFastbeltReferencesConstructor) TokenGroupTokenRefs(owner core.AstNode, unit core.StringUnit) *core.Reference[AbstractTokenRule] { fn := s.referenceLinker().LinkTokenGroupTokenRefs return core.NewReference(owner, unit, fn) } +func (s *DefaultFastbeltReferencesConstructor) TokenUsageTokenRef(owner core.AstNode, unit core.StringUnit) *core.Reference[AbstractTokenRule] { + fn := s.referenceLinker().LinkTokenUsageTokenRef + return core.NewReference(owner, unit, fn) +} + func (s *DefaultFastbeltReferencesConstructor) AssignmentProperty(owner core.AstNode, unit core.StringUnit) *core.Reference[Field] { fn := s.referenceLinker().LinkAssignmentProperty return core.NewReference(owner, unit, fn) @@ -240,6 +274,7 @@ func NewSymbolContainers() *FastbeltSymbolContainers { } type FastbeltSymbolContainer struct { + TokenModes []*core.SymbolDescription Interfaces []*core.SymbolDescription Fields []*core.SymbolDescription AbstractTokenRules []*core.SymbolDescription @@ -248,6 +283,9 @@ type FastbeltSymbolContainer struct { func (sc *FastbeltSymbolContainer) Put(desc *core.SymbolDescription) bool { switch desc.Node.(type) { + case TokenMode: + sc.TokenModes = append(sc.TokenModes, desc) + return true case Interface: sc.Interfaces = append(sc.Interfaces, desc) return true @@ -266,6 +304,7 @@ func (sc *FastbeltSymbolContainer) Put(desc *core.SymbolDescription) bool { func (sc *FastbeltSymbolContainer) All() core.SymbolSeq { return extiter.Concat( + slices.Values(sc.TokenModes), slices.Values(sc.Interfaces), slices.Values(sc.Fields), slices.Values(sc.AbstractTokenRules), @@ -273,6 +312,7 @@ func (sc *FastbeltSymbolContainer) All() core.SymbolSeq { ) } +var TypeFor_TokenMode = reflect.TypeFor[TokenMode]() var TypeFor_Interface = reflect.TypeFor[Interface]() var TypeFor_Field = reflect.TypeFor[Field]() var TypeFor_AbstractTokenRule = reflect.TypeFor[AbstractTokenRule]() @@ -280,6 +320,8 @@ var TypeFor_AbstractRule = reflect.TypeFor[AbstractRule]() func (sc *FastbeltSymbolContainer) ForType(t reflect.Type) core.SymbolSeq { switch t { + case TypeFor_TokenMode: + return slices.Values(sc.TokenModes) case TypeFor_Interface: return slices.Values(sc.Interfaces) case TypeFor_Field: diff --git a/internal/grammar/linking_test.go b/internal/grammar/linking_test.go index 40d6d9aa..f7f1ab16 100644 --- a/internal/grammar/linking_test.go +++ b/internal/grammar/linking_test.go @@ -355,3 +355,314 @@ func TestCrossDocumentReferenceLinking(t *testing.T) { assert.Equal(t, "Animal", resolvedIface.Name()) assert.Same(t, test.MustFindNamedNode[Interface](typesDoc, "Animal"), resolvedIface) } + +// --- Token mode linking and scoping --- + +func TestTokenCommandModeLinking(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(<|target:Other|>) + hidden WS + } + + token mode Other { + ID -> pop + } + `) + doc.AssertNoLinkingErrors() + modeRef := test.MustFindReference[TokenMode](doc, "target") + require.Nil(t, modeRef.Error()) + assert.Equal(t, "Other", modeRef.Ref(doc.Ctx()).Name()) +} + +func TestTokenCommandDefaultModeHasNoReference(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(Other) + hidden WS + } + + token mode Other { + ID -> mode(default) + } + `) + doc.AssertNoLinkingErrors() + // `default` is a marker on the command, not a reference, because the + // default mode has no name of its own. + commands := test.FindAll[TokenCommand](doc) + var setMode TokenCommand + for _, command := range commands { + if command.Type() == "mode" { + setMode = command + } + } + require.NotNil(t, setMode) + assert.True(t, setMode.IsDefault()) + assert.Nil(t, setMode.Mode()) +} + +func TestRuleCallToModeLocalTokenDeclaration(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string Content string } + Foo: Greeting=ID "(" Content=<|inner:INNER|> ")"; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + "(" -> push(Inner) + hidden WS + } + + token mode Inner { + token INNER: /[A-Z]+/ + ")" -> pop + } + `) + // A token declared inside a mode is visible to parser rules. + doc.AssertNoLinkingErrors() + ref := test.MustFindReference[AbstractRule](doc, "inner") + require.Nil(t, ref.Error()) + assert.Equal(t, "INNER", ref.Ref(doc.Ctx()).Name()) +} + +func TestRuleCallToModeLocalTokenGroup(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string Closer string } + Foo: Greeting=ID "(" Closer=<|closers:Closers|>; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + "(" -> push(Inner) + hidden WS + } + + token mode Inner { + token group Closers { ")" "]" } -> pop + } + `) + doc.AssertNoLinkingErrors() + ref := test.MustFindReference[AbstractRule](doc, "closers") + require.Nil(t, ref.Error()) + assert.Equal(t, "Closers", ref.Ref(doc.Ctx()).Name()) +} + +func TestModeLocalTokenIsNotVisibleToOtherModes(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string Content string } + Foo: Greeting=ID "(" Content=INNER ")"; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + "(" -> push(Inner) + hidden WS + } + + token mode Inner { + token INNER: /[A-Z]+/ + ")" -> pop + } + + token mode Other { + <|1:INNER|> -> pop + } + `) + // A mode-local declaration belongs to its own mode: parser rules can call + // it, but another mode cannot list it. + doc.ExpectDiagnostic("1").WithSeverity(core.SeverityError). + WithMessageContaining("Could not resolve reference to 'INNER'") +} + +func TestModeLocalTokenShadowsTopLevelTokenOfSameName(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=<|ref:ID|>; + + token <|1:ID|>: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + token <|2:ID|>: /[A-Z]+/ + hidden WS + } + `) + // Both declarations are reported as duplicates; the reference resolves to + // the mode-local one because it is added in front of the outer scope. + doc.ExpectDiagnostic("1").WithCode(ValidateUniqueRuleName) + doc.ExpectDiagnostic("2").WithCode(ValidateUniqueRuleName) + ref := test.MustFindReference[AbstractRule](doc, "ref") + require.Nil(t, ref.Error()) + resolved, ok := ref.Ref(doc.Ctx()).(TokenDecl) + require.True(t, ok) + _, isModeLocal := resolved.Container().(TokenDeclUsage) + assert.True(t, isModeLocal, "the mode-local declaration must win over the top-level one") +} + +func TestTokenModeNameMayEqualTokenName(t *testing.T) { + f := test.New(t, CreateServices()) + // Token modes live in their own namespace, so a mode may carry the name of + // a token without clashing. + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + token STRING: /"[^"]*"/ + hidden token WS: /\s+/ + + token mode default { + ID + STRING -> push(<|target:STRING|>) + hidden WS + } + + token mode STRING { + ID -> pop + } + `) + doc.AssertNoLinkingErrors() + modeRef := test.MustFindReference[TokenMode](doc, "target") + require.Nil(t, modeRef.Error()) + assert.Equal(t, "STRING", modeRef.Ref(doc.Ctx()).Name()) +} + +func TestTokenModeReferenceAcrossDocuments(t *testing.T) { + f := test.New(t, CreateServices()) + docs := f.ParseAll( + "inmemory://modes.fb", ` + grammar Modes; + interface Bar { Name string } + Bar: Name=NAME; + token NAME: /[A-Z]+/ + token mode default { + NAME + } + token mode Shared { + NAME -> pop + } + `, + "inmemory://main.fb", ` + grammar Main; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID -> push(<|target:Shared|>) + hidden WS + } + `, + ) + mainDoc, modesDoc := docs[1], docs[0] + // Token modes are file-local: a command cannot target a mode declared in + // another document, even though both files form one package. The generated + // lexer has one mode table per grammar and a command's target is an index + // into that table, so a mode from elsewhere cannot be represented. + modeRef := test.MustFindReference[TokenMode](mainDoc, "target") + require.NotNil(t, modeRef.Error()) + mainDoc.ExpectDiagnostic("target"). + WithSeverity(core.SeverityError). + WithMessageContaining("Could not resolve reference to 'Shared'") + // Other named nodes are still visible across the package. + modesDoc.AssertNoLinkingErrors() +} + +func TestCrossRefsOnTokenModesAreInvalid(t *testing.T) { + f := test.New(t, CreateServices()) + docs := f.ParseAll( + "inmemory://modes.fb", ` + grammar Modes; + interface Bar { Name string } + Bar: Name=NAME INNER; + token NAME: /[A-Z]+/ + token mode default { + NAME -> push(Inner) + } + token mode Inner { token INNER: /[A-Z]+/ -> pop } + `, + "inmemory://main.fb", ` + grammar Main; + interface Foo { Greeting string } + Foo: Greeting=<|target:INNER|>; + `, + ) + mainDoc, modesDoc := docs[1], docs[0] + mainDoc.ExpectDiagnostic("target"). + WithSeverity(core.SeverityError). + WithMessageContaining("Could not resolve reference to 'INNER'") + modesDoc.AssertNoLinkingErrors() +} + +func TestTokenModeReferenceResolvesWithinSameDocument(t *testing.T) { + f := test.New(t, CreateServices()) + // The counterpart to the cross-document case: a mode declared next to the + // command resolves as usual. + docs := f.ParseAll( + "inmemory://other.fb", ` + grammar Other; + interface Bar { Name string } + Bar: Name=NAME; + token NAME: /[A-Z]+/ + token mode default { + NAME + } + token mode Shared { + NAME -> pop + } + `, + "inmemory://own.fb", ` + grammar Own; + interface Foo { Greeting string } + Foo: Greeting=ID; + token ID: /[a-z]+/ + hidden token WS: /\s+/ + token mode default { + ID -> push(<|target:Shared|>) + hidden WS + } + token mode Shared { + ID -> pop + } + `, + ) + ownDoc := docs[1] + ownDoc.AssertNoLinkingErrors() + modeRef := test.MustFindReference[TokenMode](ownDoc, "target") + require.Nil(t, modeRef.Error()) + resolved := modeRef.Ref(ownDoc.Ctx()) + assert.Equal(t, "Shared", resolved.Name()) + // The local mode wins over the identically named one in the sibling file. + assert.Equal(t, ownDoc.Document.URI, resolved.Document().URI) +} diff --git a/internal/grammar/parser_gen.go b/internal/grammar/parser_gen.go index 1f58386b..6983a131 100644 --- a/internal/grammar/parser_gen.go +++ b/internal/grammar/parser_gen.go @@ -68,7 +68,7 @@ func (p *Parser) ParseGrammar() Grammar { case 1: { p.state.EnterRule(Grammar__Basic_5) - result := p.ParseToken() + result := p.ParseTokenDecl() p.state.ExitRule() if result != nil { current.SetTerminalsItem(result) @@ -86,15 +86,24 @@ func (p *Parser) ParseGrammar() Grammar { case 3: { p.state.EnterRule(Grammar__Basic_9) - result := p.ParseInterface() + result := p.ParseTokenMode() p.state.ExitRule() if result != nil { - current.SetInterfacesItem(result) + current.SetTokenModesItem(result) } } case 4: { p.state.EnterRule(Grammar__Basic_11) + result := p.ParseInterface() + p.state.ExitRule() + if result != nil { + current.SetInterfacesItem(result) + } + } + case 5: + { + p.state.EnterRule(Grammar__Basic_13) result := p.ParseCompositeRule() p.state.ExitRule() if result != nil { @@ -399,55 +408,188 @@ func (p *Parser) ParseParserRule() ParserRule { return current } -func (p *Parser) ParseToken() Token { - current := NewToken() +func (p *Parser) ParseTokenDecl() TokenDecl { + current := NewTokenDecl() current.SetTextRangeStart(p.state.LA(1).Range.Start) { - p.state.Sync(Token__Basic_3) - switch prediction, _ := p.lookahead.TokenAlternatives(p.state); prediction { - case 0: - { - token := p.state.Consume(Keyword_hidden) - core.AssignToken(current, token, Token_Type_hidden) - if token != nil { - current.SetType(token) - } - } - case 1: - { - token := p.state.Consume(Keyword_comment) - core.AssignToken(current, token, Token_Type_comment) + { + p.state.Sync(TokenDecl__Basic_2) + if p.lookahead.TokenDeclModifierOptional(p.state) { + token := p.state.Consume(TokenGroup_TokenModifier) + core.AssignToken(current, token, TokenDecl__Basic_0) if token != nil { - current.SetType(token) + current.SetModifier(token) } } } { token := p.state.Consume(Keyword_token) - core.AssignToken(current, token, Token_token) + core.AssignToken(current, token, TokenDecl_token) } { token := p.state.Consume(Token_ID) - core.AssignToken(current, token, Token_Name_ID) + core.AssignToken(current, token, TokenDecl_Name_ID) if token != nil { current.SetName(token) } } { token := p.state.Consume(Keyword_Colon) - core.AssignToken(current, token, Token_Colon) + core.AssignToken(current, token, TokenDecl_Colon) } + { + p.state.EnterRule(TokenDecl__Basic_6) + result := p.ParseTokenContent() + p.state.ExitRule() + if result != nil { + current.SetContent(result) + } + } + { + p.state.Sync(TokenDecl__Basic_6) + if p.lookahead.TokenDeclCommandOptional(p.state) { + p.state.EnterRule(TokenDecl__Basic_5) + result := p.ParseTokenCommand() + p.state.ExitRule() + if result != nil { + current.SetCommand(result) + } + } + } + { + if p.lookahead.TokenDeclSemicolonOptional(p.state) { + token := p.state.Consume(Keyword_Semicolon) + core.AssignToken(current, token, TokenDecl_Semicolon) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseTokenContent() TokenContent { + startPos := p.state.LA(1).Range.Start + var current TokenContent + { + switch prediction, failure := p.lookahead.TokenContentAlternatives(p.state); prediction { + case 0: + { + p.state.EnterRule(TokenContent__Basic_1) + result := p.ParseRegexpTokenContent() + p.state.ExitRule() + current = result + } + case 1: + { + p.state.EnterRule(TokenContent__Basic_3) + result := p.ParseKeywordTokenContent() + p.state.ExitRule() + current = result + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } + if current == nil { + current = NewTokenContent() + current.SetTextRangeStart(startPos) + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseRegexpTokenContent() RegexpTokenContent { + current := NewRegexpTokenContent() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { { token := p.state.Consume(Token_RegexLiteral) - core.AssignToken(current, token, Token_Regexp_RegexLiteral) + core.AssignToken(current, token, RegexpTokenContent_Regexp_RegexLiteral) if token != nil { current.SetRegexp(token) } } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseKeywordTokenContent() KeywordTokenContent { + current := NewKeywordTokenContent() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { { - if p.lookahead.TokenSemicolonOptional(p.state) { - token := p.state.Consume(Keyword_Semicolon) - core.AssignToken(current, token, Token_Semicolon) + p.state.EnterRule(KeywordTokenContent__Basic_1) + result := p.ParseKeyword() + p.state.ExitRule() + if result != nil { + current.SetKeyword(result) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseTokenCommand() TokenCommand { + current := NewTokenCommand() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Keyword_DashGreaterThan) + core.AssignToken(current, token, TokenCommand_DashGreaterThan) + } + { + switch prediction, _ := p.lookahead.TokenCommandTypeAlternatives(p.state); prediction { + case 0: + token := p.state.Consume(Keyword_push) + core.AssignToken(current, token, TokenCommand_Type_push) + if token != nil { + current.SetType(token) + } + case 1: + token := p.state.Consume(Keyword_pop) + core.AssignToken(current, token, TokenCommand_Type_pop) + if token != nil { + current.SetType(token) + } + case 2: + token := p.state.Consume(Keyword_mode) + core.AssignToken(current, token, TokenCommand_Type_mode) + if token != nil { + current.SetType(token) + } + } + } + p.state.Sync(TokenCommand__Basic_8) + if p.lookahead.TokenCommandOptional(p.state) { + { + token := p.state.Consume(Keyword_LeftParen) + core.AssignToken(current, token, TokenCommand_LeftParen) + } + switch prediction, failure := p.lookahead.TokenCommandAlternatives(p.state); prediction { + case 0: + { + token := p.state.Consume(Token_ID) + core.AssignToken(current, token, TokenCommand_Mode_ID) + if token != nil { + current.SetMode(p.referencesConstructor.TokenCommandMode(current, token)) + } + } + case 1: + { + token := p.state.Consume(Keyword_default) + core.AssignToken(current, token, TokenCommand_Default_default) + if token != nil { + current.SetDefault(token) + } + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } + { + token := p.state.Consume(Keyword_RightParen) + core.AssignToken(current, token, TokenCommand_RightParen) } } } @@ -459,6 +601,16 @@ func (p *Parser) ParseTokenGroup() TokenGroup { current := NewTokenGroup() current.SetTextRangeStart(p.state.LA(1).Range.Start) { + { + p.state.Sync(TokenGroup__Basic_2) + if p.lookahead.TokenGroupModifierOptional(p.state) { + token := p.state.Consume(TokenGroup_TokenModifier) + core.AssignToken(current, token, TokenGroup__Basic_0) + if token != nil { + current.SetModifier(token) + } + } + } { token := p.state.Consume(Keyword_token) core.AssignToken(current, token, TokenGroup_token) @@ -491,24 +643,24 @@ func (p *Parser) ParseTokenGroup() TokenGroup { } } case 1: + { + p.state.EnterRule(TokenGroup__Basic_5) + result := p.ParseKeyword() + p.state.ExitRule() + if result != nil { + current.SetKeywordsItem(result) + } + } + case 2: { token := p.state.Consume(Keyword_keywords) core.AssignToken(current, token, TokenGroup_keywords) } { token := p.state.Consume(Token_RegexLiteral) - core.AssignToken(current, token, TokenGroup_Regexps_RegexLiteral) + core.AssignToken(current, token, TokenGroup_KeywordSelectors_RegexLiteral) if token != nil { - current.SetRegexpsItem(token) - } - } - case 2: - { - p.state.EnterRule(TokenGroup__Basic_3) - result := p.ParseKeyword() - p.state.ExitRule() - if result != nil { - current.SetKeywordsItem(result) + current.SetKeywordSelectorsItem(token) } } default: @@ -520,6 +672,279 @@ func (p *Parser) ParseTokenGroup() TokenGroup { token := p.state.Consume(Keyword_RightBrace) core.AssignToken(current, token, TokenGroup_RightBrace) } + { + p.state.Sync(TokenGroup__Basic_10) + if p.lookahead.TokenGroupCommandOptional(p.state) { + p.state.EnterRule(TokenGroup__Basic_9) + result := p.ParseTokenCommand() + p.state.ExitRule() + if result != nil { + current.SetCommand(result) + } + } + } + { + if p.lookahead.TokenGroupSemicolonOptional(p.state) { + token := p.state.Consume(Keyword_Semicolon) + core.AssignToken(current, token, TokenGroup_Semicolon) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseTokenMode() TokenMode { + current := NewTokenMode() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Keyword_token) + core.AssignToken(current, token, TokenMode_token) + } + { + token := p.state.Consume(Keyword_mode) + core.AssignToken(current, token, TokenMode_mode) + } + switch prediction, failure := p.lookahead.TokenModeAlternatives(p.state); prediction { + case 0: + { + token := p.state.Consume(Token_ID) + core.AssignToken(current, token, TokenMode_Name_ID) + if token != nil { + current.SetName(token) + } + } + case 1: + { + token := p.state.Consume(Keyword_default) + core.AssignToken(current, token, TokenMode_Default_default) + if token != nil { + current.SetDefault(token) + } + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } + { + token := p.state.Consume(Keyword_LeftBrace) + core.AssignToken(current, token, TokenMode_LeftBrace) + } + { + p.state.Sync(TokenMode__LoopEntry) + for p.lookahead.TokenModeMembersLoop(p.state) { + p.state.EnterRule(TokenMode__Basic_4) + result := p.ParseTokenModeMember() + p.state.ExitRule() + if result != nil { + current.SetMembersItem(result) + } + p.state.Sync(TokenMode__LoopEntry) + } + } + { + token := p.state.Consume(Keyword_RightBrace) + core.AssignToken(current, token, TokenMode_RightBrace) + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseTokenModeMember() TokenModeMember { + startPos := p.state.LA(1).Range.Start + var current TokenModeMember + { + switch prediction, failure := p.lookahead.TokenModeMemberAlternatives(p.state); prediction { + case 0: + { + p.state.EnterRule(TokenModeMember__Basic_1) + result := p.ParseTokenDeclUsage() + p.state.ExitRule() + current = result + } + case 1: + { + p.state.EnterRule(TokenModeMember__Basic_3) + result := p.ParseTokenGroupUsage() + p.state.ExitRule() + current = result + } + case 2: + { + p.state.EnterRule(TokenModeMember__Basic_5) + result := p.ParseTokenUsage() + p.state.ExitRule() + current = result + } + case 3: + { + p.state.EnterRule(TokenModeMember__Basic_7) + result := p.ParseKeywordUsage() + p.state.ExitRule() + current = result + } + case 4: + { + p.state.EnterRule(TokenModeMember__Basic_9) + result := p.ParseKeywordSelector() + p.state.ExitRule() + current = result + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } + if current == nil { + current = NewTokenModeMember() + current.SetTextRangeStart(startPos) + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseTokenDeclUsage() TokenDeclUsage { + current := NewTokenDeclUsage() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + p.state.EnterRule(TokenDeclUsage__Basic_1) + result := p.ParseTokenDecl() + p.state.ExitRule() + if result != nil { + current.SetDeclaration(result) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseTokenGroupUsage() TokenGroupUsage { + current := NewTokenGroupUsage() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + p.state.EnterRule(TokenGroupUsage__Basic_1) + result := p.ParseTokenGroup() + p.state.ExitRule() + if result != nil { + current.SetGroup(result) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseTokenUsage() TokenUsage { + current := NewTokenUsage() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + p.state.Sync(TokenUsage__Basic_2) + if p.lookahead.TokenUsageModifierOptional(p.state) { + token := p.state.Consume(TokenGroup_TokenModifier) + core.AssignToken(current, token, TokenUsage__Basic_0) + if token != nil { + current.SetModifier(token) + } + } + } + { + token := p.state.Consume(Token_ID) + core.AssignToken(current, token, TokenUsage_TokenRef_ID) + if token != nil { + current.SetTokenRef(p.referencesConstructor.TokenUsageTokenRef(current, token)) + } + } + { + p.state.Sync(TokenUsage__Basic_5) + if p.lookahead.TokenUsageCommandOptional(p.state) { + p.state.EnterRule(TokenUsage__Basic_4) + result := p.ParseTokenCommand() + p.state.ExitRule() + if result != nil { + current.SetCommand(result) + } + } + } + { + if p.lookahead.TokenUsageSemicolonOptional(p.state) { + token := p.state.Consume(Keyword_Semicolon) + core.AssignToken(current, token, TokenUsage_Semicolon) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseKeywordUsage() KeywordUsage { + current := NewKeywordUsage() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + p.state.Sync(KeywordUsage__Basic_2) + if p.lookahead.KeywordUsageModifierOptional(p.state) { + token := p.state.Consume(TokenGroup_TokenModifier) + core.AssignToken(current, token, KeywordUsage__Basic_0) + if token != nil { + current.SetModifier(token) + } + } + } + { + p.state.EnterRule(KeywordUsage__Basic_6) + result := p.ParseKeyword() + p.state.ExitRule() + if result != nil { + current.SetKeyword(result) + } + } + { + p.state.Sync(KeywordUsage__Basic_6) + if p.lookahead.KeywordUsageCommandOptional(p.state) { + p.state.EnterRule(KeywordUsage__Basic_5) + result := p.ParseTokenCommand() + p.state.ExitRule() + if result != nil { + current.SetCommand(result) + } + } + } + { + if p.lookahead.KeywordUsageSemicolonOptional(p.state) { + token := p.state.Consume(Keyword_Semicolon) + core.AssignToken(current, token, KeywordUsage_Semicolon) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseKeywordSelector() KeywordSelector { + current := NewKeywordSelector() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Keyword_keywords) + core.AssignToken(current, token, KeywordSelector_keywords) + } + { + token := p.state.Consume(Token_RegexLiteral) + core.AssignToken(current, token, KeywordSelector_Selector_RegexLiteral) + if token != nil { + current.SetSelector(token) + } + } + { + if p.lookahead.KeywordSelectorSemicolonOptional(p.state) { + token := p.state.Consume(Keyword_Semicolon) + core.AssignToken(current, token, KeywordSelector_Semicolon) + } + } } current.SetTextRangeEnd(p.state.LA(0).Range.End) return current @@ -660,23 +1085,10 @@ func (p *Parser) ParseElement() Element { current.SetTextRangeStart(startPos) } { - p.state.Sync(Element__Basic_15) - switch prediction, _ := p.lookahead.ElementCardinalityAlternatives(p.state); prediction { - case 0: - token := p.state.Consume(Keyword_Asterisk) - core.AssignToken(current, token, Element_Cardinality_Asterisk) - if token != nil { - current.SetCardinality(token) - } - case 1: - token := p.state.Consume(Keyword_Plus) - core.AssignToken(current, token, Element_Cardinality_Plus) - if token != nil { - current.SetCardinality(token) - } - case 2: - token := p.state.Consume(Keyword_Question) - core.AssignToken(current, token, Element_Cardinality_Question) + p.state.Sync(Element__Basic_13) + if p.lookahead.ElementCardinalityOptional(p.state) { + token := p.state.Consume(TokenGroup_Cardinality) + core.AssignToken(current, token, Element__Basic_11) if token != nil { current.SetCardinality(token) } @@ -1155,23 +1567,10 @@ func (p *Parser) ParseCompositeElement() Element { current.SetTextRangeStart(startPos) } { - p.state.Sync(CompositeElement__Basic_11) - switch prediction, _ := p.lookahead.CompositeElementCardinalityAlternatives(p.state); prediction { - case 0: - token := p.state.Consume(Keyword_Asterisk) - core.AssignToken(current, token, CompositeElement_Cardinality_Asterisk) - if token != nil { - current.SetCardinality(token) - } - case 1: - token := p.state.Consume(Keyword_Plus) - core.AssignToken(current, token, CompositeElement_Cardinality_Plus) - if token != nil { - current.SetCardinality(token) - } - case 2: - token := p.state.Consume(Keyword_Question) - core.AssignToken(current, token, CompositeElement_Cardinality_Question) + p.state.Sync(CompositeElement__Basic_9) + if p.lookahead.CompositeElementCardinalityOptional(p.state) { + token := p.state.Consume(TokenGroup_Cardinality) + core.AssignToken(current, token, CompositeElement__Basic_7) if token != nil { current.SetCardinality(token) } diff --git a/internal/grammar/parser_lookahead_gen.go b/internal/grammar/parser_lookahead_gen.go index ccb31d10..1719d66d 100644 --- a/internal/grammar/parser_lookahead_gen.go +++ b/internal/grammar/parser_lookahead_gen.go @@ -8,72 +8,83 @@ import ( ) const ( - DecisionCompositeGroupElementsLoop = 34 - DecisionCompositeGroupOptional = 35 - DecisionElementAlternatives = 20 - DecisionGrammarAlternatives = 1 - DecisionGroupElementsLoop = 18 - DecisionGroupOptional = 19 + DecisionCompositeGroupElementsLoop = 50 + DecisionCompositeGroupOptional = 51 + DecisionElementAlternatives = 37 + DecisionGrammarAlternatives = 1 + DecisionGroupElementsLoop = 35 + DecisionGroupOptional = 36 + DecisionTokenModeMemberAlternatives = 25 ) var ActionOperatorAlternatives = parser.LL1Lookahead{ Types: []*core.TokenType{Keyword_PlusEquals, Keyword_Equals}, - Lookup: []int{5: 1, 10: 2}, + Lookup: []int{5: 1, 11: 2}, } var AssignableAlternatives = parser.LL1Lookahead{ Types: []*core.TokenType{Token_StringLiteral, Token_ID, Keyword_LeftBracket, Keyword_LeftParen}, - Lookup: []int{1: 4, 13: 3, 34: 1, 35: 2}, + Lookup: []int{1: 4, 14: 3, 39: 1, 40: 2}, } var AssignableWithoutAltsAlternatives = parser.LL1Lookahead{ Types: []*core.TokenType{Token_StringLiteral, Token_ID, Keyword_LeftBracket}, - Lookup: []int{13: 3, 34: 1, 35: 2}, + Lookup: []int{14: 3, 39: 1, 40: 2}, } var AssignmentOperatorAlternatives = parser.LL1Lookahead{ Types: []*core.TokenType{Keyword_PlusEquals, Keyword_Equals, Keyword_QuestionEquals}, - Lookup: []int{5: 1, 10: 2, 12: 3}, + Lookup: []int{5: 1, 11: 2, 13: 3}, } var CompositeElementAlternatives = parser.LL1Lookahead{ Types: []*core.TokenType{Token_StringLiteral, Token_ID, Keyword_LeftParen}, - Lookup: []int{1: 3, 34: 1, 35: 2}, -} - -var CompositeElementCardinalityAlternatives = parser.LL1Lookahead{ - Types: []*core.TokenType{Keyword_Asterisk, Keyword_Plus, Keyword_Question}, - Lookup: []int{3: 1, 4: 2, 11: 3}, -} - -var ElementCardinalityAlternatives = parser.LL1Lookahead{ - Types: []*core.TokenType{Keyword_Asterisk, Keyword_Plus, Keyword_Question}, - Lookup: []int{3: 1, 4: 2, 11: 3}, + Lookup: []int{1: 3, 39: 1, 40: 2}, } var FieldTypeAlternatives = parser.LL1Lookahead{ Types: []*core.TokenType{Token_ID, Keyword_Asterisk, Keyword_LeftBracket, Keyword_bool, Keyword_composite, Keyword_string}, - Lookup: []int{3: 2, 13: 3, 15: 4, 17: 4, 27: 4, 35: 1}, + Lookup: []int{3: 2, 14: 3, 16: 4, 18: 4, 32: 4, 40: 1}, } var GrammarLoop = parser.LL1Lookahead{ - Types: []*core.TokenType{Keyword_comment, Keyword_composite, Keyword_entry, Keyword_hidden, Keyword_interface, Keyword_token, Token_ID}, - Lookup: []int{16: 1, 17: 1, 19: 1, 23: 1, 24: 1, 28: 1, 35: 1}, + Types: []*core.TokenType{Keyword_composite, Keyword_entry, Keyword_interface, Keyword_token, Token_ID, TokenGroup_TokenModifier}, + Lookup: []int{17: 1, 18: 1, 21: 1, 25: 1, 26: 1, 33: 1, 40: 1}, } var PrimitiveTypeTypeAlternatives = parser.LL1Lookahead{ Types: []*core.TokenType{Keyword_string, Keyword_bool, Keyword_composite}, - Lookup: []int{15: 2, 17: 3, 27: 1}, + Lookup: []int{16: 2, 18: 3, 32: 1}, +} + +var TokenCommandAlternatives = parser.LL1Lookahead{ + Types: []*core.TokenType{Token_ID, Keyword_default}, + Lookup: []int{20: 2, 40: 1}, +} + +var TokenCommandTypeAlternatives = parser.LL1Lookahead{ + Types: []*core.TokenType{Keyword_push, Keyword_pop, Keyword_mode}, + Lookup: []int{28: 3, 29: 2, 30: 1}, } -var TokenAlternatives = parser.LL1Lookahead{ - Types: []*core.TokenType{Keyword_hidden, Keyword_comment}, - Lookup: []int{16: 2, 23: 1}, +var TokenContentAlternatives = parser.LL1Lookahead{ + Types: []*core.TokenType{Token_RegexLiteral, Token_StringLiteral}, + Lookup: []int{39: 2, 41: 1}, } var TokenGroupAlternatives = parser.LL1Lookahead{ - Types: []*core.TokenType{Token_ID, Keyword_keywords, Token_StringLiteral}, - Lookup: []int{25: 2, 34: 3, 35: 1}, + Types: []*core.TokenType{Token_ID, Token_StringLiteral, Keyword_keywords}, + Lookup: []int{27: 3, 39: 2, 40: 1}, +} + +var TokenModeAlternatives = parser.LL1Lookahead{ + Types: []*core.TokenType{Token_ID, Keyword_default}, + Lookup: []int{20: 2, 40: 1}, +} + +var TokenModeMembersLoop = parser.LL1Lookahead{ + Types: []*core.TokenType{Keyword_keywords, Keyword_token, Token_StringLiteral, Token_ID, TokenGroup_TokenModifier}, + Lookup: []int{17: 1, 25: 1, 27: 1, 33: 1, 39: 1, 40: 1}, } // FastbeltParserLookahead abstracts every lookahead/prediction decision performed by @@ -95,13 +106,13 @@ type FastbeltParserLookahead interface { CompositeAlternativesLoop(state *parser.ParserState) bool CompositeAlternativesOptional(state *parser.ParserState) bool CompositeElementAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) - CompositeElementCardinalityAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + CompositeElementCardinalityOptional(state *parser.ParserState) bool CompositeGroupElementsLoop(state *parser.ParserState) bool CompositeGroupOptional(state *parser.ParserState) bool CompositeRuleSemicolonOptional(state *parser.ParserState) bool CrossRefOptional(state *parser.ParserState) bool ElementAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) - ElementCardinalityAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + ElementCardinalityOptional(state *parser.ParserState) bool FieldTypeAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) GrammarAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) GrammarLoop(state *parser.ParserState) bool @@ -111,13 +122,31 @@ type FastbeltParserLookahead interface { InterfaceFieldsLoop(state *parser.ParserState) bool InterfaceLoop(state *parser.ParserState) bool InterfaceOptional(state *parser.ParserState) bool + KeywordSelectorSemicolonOptional(state *parser.ParserState) bool + KeywordUsageCommandOptional(state *parser.ParserState) bool + KeywordUsageModifierOptional(state *parser.ParserState) bool + KeywordUsageSemicolonOptional(state *parser.ParserState) bool ParserRuleEntryOptional(state *parser.ParserState) bool ParserRuleOptional(state *parser.ParserState) bool ParserRuleSemicolonOptional(state *parser.ParserState) bool PrimitiveTypeTypeAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) - TokenAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + TokenCommandAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + TokenCommandOptional(state *parser.ParserState) bool + TokenCommandTypeAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + TokenContentAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + TokenDeclCommandOptional(state *parser.ParserState) bool + TokenDeclModifierOptional(state *parser.ParserState) bool + TokenDeclSemicolonOptional(state *parser.ParserState) bool TokenGroupAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) - TokenSemicolonOptional(state *parser.ParserState) bool + TokenGroupCommandOptional(state *parser.ParserState) bool + TokenGroupModifierOptional(state *parser.ParserState) bool + TokenGroupSemicolonOptional(state *parser.ParserState) bool + TokenModeAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + TokenModeMemberAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + TokenModeMembersLoop(state *parser.ParserState) bool + TokenUsageCommandOptional(state *parser.ParserState) bool + TokenUsageModifierOptional(state *parser.ParserState) bool + TokenUsageSemicolonOptional(state *parser.ParserState) bool } // DefaultFastbeltParserLookahead resolves every decision with the parser state's built-in @@ -178,8 +207,8 @@ func (l *DefaultFastbeltParserLookahead) CompositeElementAlternatives(state *par return state.Lookahead(CompositeElementAlternatives) } -func (l *DefaultFastbeltParserLookahead) CompositeElementCardinalityAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { - return state.Lookahead(CompositeElementCardinalityAlternatives) +func (l *DefaultFastbeltParserLookahead) CompositeElementCardinalityOptional(state *parser.ParserState) bool { + return TokenGroup_Cardinality.Matches(state.LA(1).Type) } func (l *DefaultFastbeltParserLookahead) CompositeGroupElementsLoop(state *parser.ParserState) bool { @@ -204,8 +233,8 @@ func (l *DefaultFastbeltParserLookahead) ElementAlternatives(state *parser.Parse return state.AdaptivePredict(DecisionElementAlternatives, l.PredictionMode()) } -func (l *DefaultFastbeltParserLookahead) ElementCardinalityAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { - return state.Lookahead(ElementCardinalityAlternatives) +func (l *DefaultFastbeltParserLookahead) ElementCardinalityOptional(state *parser.ParserState) bool { + return TokenGroup_Cardinality.Matches(state.LA(1).Type) } func (l *DefaultFastbeltParserLookahead) FieldTypeAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { @@ -247,6 +276,22 @@ func (l *DefaultFastbeltParserLookahead) InterfaceOptional(state *parser.ParserS return state.LA(1).Type == Keyword_extends } +func (l *DefaultFastbeltParserLookahead) KeywordSelectorSemicolonOptional(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_Semicolon +} + +func (l *DefaultFastbeltParserLookahead) KeywordUsageCommandOptional(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_DashGreaterThan +} + +func (l *DefaultFastbeltParserLookahead) KeywordUsageModifierOptional(state *parser.ParserState) bool { + return TokenGroup_TokenModifier.Matches(state.LA(1).Type) +} + +func (l *DefaultFastbeltParserLookahead) KeywordUsageSemicolonOptional(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_Semicolon +} + func (l *DefaultFastbeltParserLookahead) ParserRuleEntryOptional(state *parser.ParserState) bool { return state.LA(1).Type == Keyword_entry } @@ -263,14 +308,71 @@ func (l *DefaultFastbeltParserLookahead) PrimitiveTypeTypeAlternatives(state *pa return state.Lookahead(PrimitiveTypeTypeAlternatives) } -func (l *DefaultFastbeltParserLookahead) TokenAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { - return state.Lookahead(TokenAlternatives) +func (l *DefaultFastbeltParserLookahead) TokenCommandAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { + return state.Lookahead(TokenCommandAlternatives) +} + +func (l *DefaultFastbeltParserLookahead) TokenCommandOptional(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_LeftParen +} + +func (l *DefaultFastbeltParserLookahead) TokenCommandTypeAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { + return state.Lookahead(TokenCommandTypeAlternatives) +} + +func (l *DefaultFastbeltParserLookahead) TokenContentAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { + return state.Lookahead(TokenContentAlternatives) +} + +func (l *DefaultFastbeltParserLookahead) TokenDeclCommandOptional(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_DashGreaterThan +} + +func (l *DefaultFastbeltParserLookahead) TokenDeclModifierOptional(state *parser.ParserState) bool { + return TokenGroup_TokenModifier.Matches(state.LA(1).Type) +} + +func (l *DefaultFastbeltParserLookahead) TokenDeclSemicolonOptional(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_Semicolon } func (l *DefaultFastbeltParserLookahead) TokenGroupAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { return state.Lookahead(TokenGroupAlternatives) } -func (l *DefaultFastbeltParserLookahead) TokenSemicolonOptional(state *parser.ParserState) bool { +func (l *DefaultFastbeltParserLookahead) TokenGroupCommandOptional(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_DashGreaterThan +} + +func (l *DefaultFastbeltParserLookahead) TokenGroupModifierOptional(state *parser.ParserState) bool { + return TokenGroup_TokenModifier.Matches(state.LA(1).Type) +} + +func (l *DefaultFastbeltParserLookahead) TokenGroupSemicolonOptional(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_Semicolon +} + +func (l *DefaultFastbeltParserLookahead) TokenModeAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { + return state.Lookahead(TokenModeAlternatives) +} + +func (l *DefaultFastbeltParserLookahead) TokenModeMemberAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { + return state.AdaptivePredict(DecisionTokenModeMemberAlternatives, l.PredictionMode()) +} + +func (l *DefaultFastbeltParserLookahead) TokenModeMembersLoop(state *parser.ParserState) bool { + prediction, _ := state.Lookahead(TokenModeMembersLoop) + return prediction == 0 +} + +func (l *DefaultFastbeltParserLookahead) TokenUsageCommandOptional(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_DashGreaterThan +} + +func (l *DefaultFastbeltParserLookahead) TokenUsageModifierOptional(state *parser.ParserState) bool { + return TokenGroup_TokenModifier.Matches(state.LA(1).Type) +} + +func (l *DefaultFastbeltParserLookahead) TokenUsageSemicolonOptional(state *parser.ParserState) bool { return state.LA(1).Type == Keyword_Semicolon } diff --git a/internal/grammar/scope_provider.go b/internal/grammar/scope_provider.go index c4e20839..04cfd480 100644 --- a/internal/grammar/scope_provider.go +++ b/internal/grammar/scope_provider.go @@ -8,6 +8,7 @@ import ( "context" core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/linking" "typefox.dev/fastbelt/util/service" ) @@ -19,6 +20,54 @@ func newScopeProviderImpl(_ *service.Container) *scopeProviderImpl { return &scopeProviderImpl{} } +// ScopeTokenCommandMode limits the target of a `push` or `mode` command to the +// token modes declared in the same document. Other named nodes are visible +// across every grammar file of a folder, but token modes are not: the generated +// lexer holds one mode table per grammar and a command's target is an index into +// that table, so a mode declared elsewhere cannot be represented. +func (s *scopeProviderImpl) ScopeTokenCommandMode(_ context.Context, reference *core.Reference[TokenMode]) core.Scope { + root, ok := reference.Owner().Document().Root.(Grammar) + if !ok { + return core.EmptyScope + } + symbols := []*core.SymbolDescription{} + for _, tokenMode := range root.TokenModes() { + // The default mode is targeted as `push(default)` rather than by name. + if tokenMode.NameToken() == nil { + continue + } + symbols = append(symbols, core.NewSymbolDescription(tokenMode, tokenMode.NameToken())) + } + return core.NewMapScopeFromSlice(symbols, nil) +} + +func (s *scopeProviderImpl) ScopeRuleCallRule(ctx context.Context, reference *core.Reference[AbstractRule]) core.Scope { + root, ok := reference.Owner().Document().Root.(Grammar) + if !ok { + return core.EmptyScope + } + symbols := []*core.SymbolDescription{} + for _, tokenMode := range root.TokenModes() { + for _, member := range tokenMode.Members() { + if tokenDeclUsage, ok := member.(TokenDeclUsage); ok { + tokenDecl := tokenDeclUsage.Declaration() + if tokenDecl.NameToken() == nil { + continue + } + symbols = append(symbols, core.NewSymbolDescription(tokenDecl, tokenDecl.NameToken())) + } else if tokenGroupUsage, ok := member.(TokenGroupUsage); ok { + tokenGroup := tokenGroupUsage.Group() + if tokenGroup.NameToken() == nil { + continue + } + symbols = append(symbols, core.NewSymbolDescription(tokenGroup, tokenGroup.NameToken())) + } + } + } + outer := linking.DefaultScopeOfType[AbstractRule](reference.Owner()) + return core.NewMapScopeFromSlice(symbols, outer) +} + func (s *scopeProviderImpl) ScopeActionProperty(ctx context.Context, reference *core.Reference[Field]) core.Scope { if action, ok := reference.Owner().(Action); ok && action.Type() != nil { targetType := action.Type().Ref(ctx) diff --git a/internal/grammar/symbol_kinds.go b/internal/grammar/symbol_kinds.go index 394ab13a..f3b58758 100644 --- a/internal/grammar/symbol_kinds.go +++ b/internal/grammar/symbol_kinds.go @@ -28,6 +28,17 @@ func (f *FieldImpl) SymbolKind() lsp.SymbolKind { return lsp.Property } -func (t *TokenImpl) SymbolKind() lsp.SymbolKind { +func (t *TokenDeclImpl) SymbolKind() lsp.SymbolKind { return lsp.Constant } + +// A token group is a named set of token constants. +func (t *TokenGroupImpl) SymbolKind() lsp.SymbolKind { + return lsp.Enum +} + +// A token mode is a named scope holding the tokens the lexer recognizes while +// the mode is active. +func (t *TokenModeImpl) SymbolKind() lsp.SymbolKind { + return lsp.Namespace +} diff --git a/internal/grammar/token_mode_lsp_test.go b/internal/grammar/token_mode_lsp_test.go new file mode 100644 index 00000000..53c57575 --- /dev/null +++ b/internal/grammar/token_mode_lsp_test.go @@ -0,0 +1,414 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package grammar + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "typefox.dev/fastbelt/server" + "typefox.dev/fastbelt/test" + "typefox.dev/fastbelt/util/service" + "typefox.dev/lsp" +) + +// tokenModeMarking replaces the fixture's `>` index-marker terminator with `@>`, +// so that the `>` of a `->` command inside a marked range is not mistaken for +// the end of a marker. Markers still must not be nested. +var tokenModeMarking = &test.TestMarking{ + StartRange: "<|", + EndRange: "|>", + StartIndex: "", + EndIndex: "@>", + Delimiter: ":", +} + +// lspFixture returns a fixture with the grammar language services, the generated +// server services (needed for completion) and the default LSP providers. +func lspFixture(t *testing.T) *test.Fixture { + sc := service.NewContainer() + SetupServices(sc) + SetupGeneratedServerServices(sc) + server.SetupDefaultServices(sc) + sc.Seal() + return test.New(t, sc).WithMarking(tokenModeMarking) +} + +func completionLabels(items []lsp.CompletionItem) []string { + labels := make([]string, len(items)) + for i, item := range items { + labels[i] = item.Label + } + return labels +} + +// --- Go to definition --- + +func TestDefinitionOnTokenModeInPushCommand(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(<|Inner@>Inner) + hidden WS +} +<|Inner:token mode Inner { + ID -> pop +}|>`, "file:///definition.fb") + doc.AssertNoErrors() + // The definition of a mode reference is the mode declaration. + doc.AssertDefinition("Inner") +} + +func TestDefinitionOnTokenModeInSetModeCommand(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> mode(<|Inner@>Inner) + hidden WS +} +<|Inner:token mode Inner { + ID -> pop +}|>`, "file:///definition-set.fb") + doc.AssertNoErrors() + doc.AssertDefinition("Inner") +} + +// --- Find references --- + +func TestReferencesOnTokenModeDeclaration(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(<|Inner|>) + hidden WS +} +token mode <|Inner@><|Inner|> { + ID -> pop +}`, "file:///references.fb") + doc.AssertNoErrors() + doc.AssertReferences("Inner") +} + +func TestReferencesOnTokenModeFromUsage(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +token NAME: /[A-Z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(<|Inner@><|Inner|>) + NAME -> push(<|Inner|>) + hidden WS +} +token mode <|Inner|> { + ID -> pop +}`, "file:///references-usage.fb") + doc.AssertNoErrors() + doc.AssertReferences("Inner") +} + +// --- Rename --- + +func TestRenameTokenMode(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(Inner) + hidden WS +} +token mode <|target@>Inner { + ID -> pop +}`, "file:///rename.fb") + doc.AssertNoErrors() + doc.RunRename("target", "Renamed") + + text := doc.Document.TextDoc.Text(nil) + assert.Contains(t, text, "token mode Renamed {") + assert.Contains(t, text, "push(Renamed)") + assert.NotContains(t, text, "Inner") + doc.AssertNoErrors() +} + +func TestRenameTokenModeFromUsage(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(<|target@>Inner) + hidden WS +} +token mode Inner { + ID -> pop +}`, "file:///rename-usage.fb") + doc.AssertNoErrors() + doc.RunRename("target", "Renamed") + + text := doc.Document.TextDoc.Text(nil) + assert.Contains(t, text, "token mode Renamed {") + assert.Contains(t, text, "push(Renamed)") + doc.AssertNoErrors() +} + +// --- Document symbols --- + +func TestDocumentSymbolForTokenMode(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID + hidden WS +} +<|mode:token mode Inner { + ID +}|>`, "file:///symbols-mode.fb") + doc.AssertNoParseErrors() + doc.AssertDocumentSymbol("mode", "Inner", lsp.Namespace) +} + +func TestDocumentSymbolForTokenGroup(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=Punct; +<|group:token group Punct { "!" }|> +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + Punct + hidden WS +}`, "file:///symbols-group.fb") + doc.AssertNoParseErrors() + doc.AssertDocumentSymbol("group", "Punct", lsp.Enum) +} + +func TestDocumentSymbolForModeLocalTokenDeclaration(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID + hidden WS +} +token mode Inner { + <|inner:token INNER: /[A-Z]+/|> +}`, "file:///symbols-inner.fb") + doc.AssertNoParseErrors() + // A token declared inside a mode is a symbol in its own right, nested under + // the mode it belongs to. + doc.AssertDocumentSymbol("inner", "INNER", lsp.Constant) + symbol := doc.MustFindSymbolAtLabel("inner") + assert.Equal(t, "INNER", symbol.Name) +} + +func TestDocumentSymbolSkipsDefaultTokenMode(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID + hidden WS +}`, "file:///symbols-default.fb") + doc.AssertNoParseErrors() + // The default mode has no name of its own, so it contributes no symbol. + provider := service.MustGet[server.DocumentSymbolProvider](f.Services()) + result, err := provider.HandleDocumentSymbolRequest(f.Ctx(), &lsp.DocumentSymbolParams{ + TextDocument: lsp.TextDocumentIdentifier{URI: doc.Document.URI.DocumentURI()}, + }) + assert.NoError(t, err) + for _, symbol := range result { + assert.NotEqual(t, "default", symbol.Name) + for _, child := range symbol.Children { + assert.NotEqual(t, "default", child.Name) + } + } +} + +// --- Folding ranges --- + +func TestFoldingRangeForTokenMode(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +<|default:token mode default { + ID -> push(Inner) + hidden WS|> +} +<|inner:token mode Inner { + ID + ID -> pop|> +}`, "file:///folding.fb") + doc.AssertNoParseErrors() + doc.AssertFoldingRanges("default", "inner") +} + +// --- Completion --- + +func TestCompletionOfTokenModeNamesInPushCommand(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(<|cursor@>) + hidden WS +} +token mode Inner { + ID -> pop +} +token mode Other { + ID -> pop +}`, "file:///completion-mode.fb") + labels := completionLabels(doc.CompletionItems("cursor")) + assert.Contains(t, labels, "Inner") + assert.Contains(t, labels, "Other") + // The default mode can be targeted by name as well. + assert.Contains(t, labels, "default") +} + +func TestCompletionOfTokenModeNamesInSetModeCommand(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(Inner) + hidden WS +} +token mode Inner { + ID -> mode(<|cursor@>) +}`, "file:///completion-set.fb") + labels := completionLabels(doc.CompletionItems("cursor")) + assert.Contains(t, labels, "default") + assert.Contains(t, labels, "Inner") +} + +func TestCompletionOfCommandKeywords(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> <|cursor@> + hidden WS +}`, "file:///completion-command.fb") + labels := completionLabels(doc.CompletionItems("cursor")) + assert.Contains(t, labels, "push") + assert.Contains(t, labels, "pop") + assert.Contains(t, labels, "mode") +} + +// --- Hover --- + +func TestHoverOnTokenModeDeclaration(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(Inner) + hidden WS +} +// The inner mode +token mode <|inner:Inner|> { + ID -> pop +}`, "file:///hover.fb") + doc.AssertNoErrors() + doc.ExpectHoverAt("inner", "The inner mode") +} + +func TestHoverOnTokenModeReference(t *testing.T) { + f := lspFixture(t) + doc := f.ParseURI(`grammar Test; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(<|usage:Inner|>) + hidden WS +} +// The inner mode +token mode Inner { + ID -> pop +}`, "file:///hover-usage.fb") + doc.AssertNoErrors() + // Hovering a reference shows the documentation of the mode it points to. + doc.ExpectHoverAt("usage", "The inner mode") +} + +func TestCompletionDoesNotOfferTokenModesFromOtherDocuments(t *testing.T) { + f := lspFixture(t) + docs := f.ParseAll( + "file:///sibling.fb", `grammar Sibling; +interface Bar { Name string } +Bar: Name=NAME; +token NAME: /[A-Z]+/ +token mode default { + NAME +} +token mode Sibling { + NAME -> pop +}`, + "file:///own.fb", `grammar Own; +interface Foo { Greeting string } +Foo: Greeting=ID; +token ID: /[a-z]+/ +hidden token WS: /\s+/ +token mode default { + ID -> push(<|cursor@>) + hidden WS +} +token mode Local { + ID -> pop +}`, + ) + labels := completionLabels(docs[1].CompletionItems("cursor")) + // Token modes are file-local, so only modes of this document are offered. + assert.Contains(t, labels, "Local") + assert.NotContains(t, labels, "Sibling", + "a mode from another document would not resolve, so it must not be offered") +} diff --git a/internal/grammar/types_gen.go b/internal/grammar/types_gen.go index a56ba3ae..665674a7 100644 --- a/internal/grammar/types_gen.go +++ b/internal/grammar/types_gen.go @@ -20,10 +20,12 @@ type Grammar interface { SetRulesItem(item ParserRule) Composites() []CompositeRule SetCompositesItem(item CompositeRule) - Terminals() []Token - SetTerminalsItem(item Token) + Terminals() []TokenDecl + SetTerminalsItem(item TokenDecl) TokenGroups() []TokenGroup SetTokenGroupsItem(item TokenGroup) + TokenModes() []TokenMode + SetTokenModesItem(item TokenMode) Interfaces() []Interface SetInterfacesItem(item Interface) } @@ -36,8 +38,9 @@ type GrammarData struct { name *core.Token rules []ParserRule composites []CompositeRule - terminals []Token + terminals []TokenDecl tokenGroups []TokenGroup + tokenModes []TokenMode interfaces []Interface } @@ -56,6 +59,9 @@ func (i *GrammarData) ForEachNode(fn func(core.AstNode, unique.Handle[string], i for j, item := range i.tokenGroups { fn(item, fieldNameTokenGroups, j) } + for j, item := range i.tokenModes { + fn(item, fieldNameTokenModes, j) + } for j, item := range i.interfaces { fn(item, fieldNameInterfaces, j) } @@ -96,11 +102,11 @@ func (i *GrammarData) SetCompositesItem(item CompositeRule) { i.composites = append(i.composites, item) } -func (i *GrammarData) Terminals() []Token { +func (i *GrammarData) Terminals() []TokenDecl { return i.terminals } -func (i *GrammarData) SetTerminalsItem(item Token) { +func (i *GrammarData) SetTerminalsItem(item TokenDecl) { i.terminals = append(i.terminals, item) } @@ -112,6 +118,14 @@ func (i *GrammarData) SetTokenGroupsItem(item TokenGroup) { i.tokenGroups = append(i.tokenGroups, item) } +func (i *GrammarData) TokenModes() []TokenMode { + return i.tokenModes +} + +func (i *GrammarData) SetTokenModesItem(item TokenMode) { + i.tokenModes = append(i.tokenModes, item) +} + func (i *GrammarData) Interfaces() []Interface { return i.interfaces } @@ -194,6 +208,17 @@ func (i *GrammarImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { return nil, fmt.Errorf("GrammarImpl.Resolve: item %d of slice in field 'tokenGroups' is nil in node '%s'", index, nodePath) } return child.Resolve(path.Tail()) + case fieldNameTokenModes: + if index >= len(i.TokenModes()) { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("GrammarImpl.Resolve: index %d exceeds length of slice in 'tokenModes' (length=%d) in node '%s'", index, len(i.TokenModes()), nodePath) + } + child := i.TokenModes()[index] + if child == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("GrammarImpl.Resolve: item %d of slice in field 'tokenModes' is nil in node '%s'", index, nodePath) + } + return child.Resolve(path.Tail()) case fieldNameName: return nil, fmt.Errorf("GrammarImpl.Resolve: field 'name' holds a primitive value instead of an ast node") default: @@ -884,6 +909,11 @@ type AbstractTokenRule interface { AbstractRule IsAbstractTokenRule() + Modifier() string + ModifierToken() *core.Token + SetModifier(value *core.Token) + Command() TokenCommand + SetCommand(value TokenCommand) } func NewAbstractTokenRule() AbstractTokenRule { @@ -891,16 +921,49 @@ func NewAbstractTokenRule() AbstractTokenRule { } type AbstractTokenRuleData struct { + modifier *core.Token + command TokenCommand } func (i *AbstractTokenRuleData) IsAbstractTokenRule() {} func (i *AbstractTokenRuleData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.command != nil { + fn(i.command, fieldNameCommand, -1) + } } func (i *AbstractTokenRuleData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { } +func (i *AbstractTokenRuleData) Modifier() string { + if i != nil && i.modifier != nil { + return i.modifier.Image + } else { + return "" + } +} + +func (i *AbstractTokenRuleData) ModifierToken() *core.Token { + return i.modifier +} + +func (i *AbstractTokenRuleData) SetModifier(value *core.Token) { + i.modifier = value +} + +func (i *AbstractTokenRuleData) Command() TokenCommand { + if i != nil && i.command != nil { + return i.command + } else { + return nil + } +} + +func (i *AbstractTokenRuleData) SetCommand(value TokenCommand) { + i.command = value +} + type AbstractTokenRuleImpl struct { core.AstNodeBase AbstractRuleData @@ -923,6 +986,15 @@ func (i *AbstractTokenRuleImpl) Resolve(path core.FragmentPath) (core.AstNode, e } field, _ := path.Head() switch field { + case fieldNameCommand: + if i.Command() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("AbstractTokenRuleImpl.Resolve: field 'command' is nil in node '%s'", nodePath) + } + child := i.Command() + return child.Resolve(path.Tail()) + case fieldNameModifier: + return nil, fmt.Errorf("AbstractTokenRuleImpl.Resolve: field 'modifier' holds a primitive value instead of an ast node") case fieldNameName: return nil, fmt.Errorf("AbstractTokenRuleImpl.Resolve: field 'name' holds a primitive value instead of an ast node") default: @@ -1031,37 +1103,321 @@ func (i *ParserRuleImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { } } -type Token interface { +type TokenDecl interface { core.AstNode AbstractTokenRule - IsToken() - Type() string - TypeToken() *core.Token - SetType(value *core.Token) + IsTokenDecl() + Content() TokenContent + SetContent(value TokenContent) +} + +func NewTokenDecl() TokenDecl { + return &TokenDeclImpl{} +} + +type TokenDeclData struct { + content TokenContent +} + +func (i *TokenDeclData) IsTokenDecl() {} + +func (i *TokenDeclData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.content != nil { + fn(i.content, fieldNameContent, -1) + } +} + +func (i *TokenDeclData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *TokenDeclData) Content() TokenContent { + if i != nil && i.content != nil { + return i.content + } else { + return nil + } +} + +func (i *TokenDeclData) SetContent(value TokenContent) { + i.content = value +} + +type TokenDeclImpl struct { + core.AstNodeBase + AbstractTokenRuleData + AbstractRuleData + TokenDeclData +} + +func (i *TokenDeclImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.AbstractTokenRuleData.ForEachNode(fn) + i.AbstractRuleData.ForEachNode(fn) + i.TokenDeclData.ForEachNode(fn) +} + +func (i *TokenDeclImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.AbstractTokenRuleData.ForEachReference(fn) + i.AbstractRuleData.ForEachReference(fn) + i.TokenDeclData.ForEachReference(fn) +} + +func (i *TokenDeclImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameCommand: + if i.Command() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenDeclImpl.Resolve: field 'command' is nil in node '%s'", nodePath) + } + child := i.Command() + return child.Resolve(path.Tail()) + case fieldNameContent: + if i.Content() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenDeclImpl.Resolve: field 'content' is nil in node '%s'", nodePath) + } + child := i.Content() + return child.Resolve(path.Tail()) + case fieldNameModifier: + return nil, fmt.Errorf("TokenDeclImpl.Resolve: field 'modifier' holds a primitive value instead of an ast node") + case fieldNameName: + return nil, fmt.Errorf("TokenDeclImpl.Resolve: field 'name' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenDeclImpl.Resolve: field '%s' does not exist in node '%s' of type 'TokenDecl'", field.Value(), nodePath) + } +} + +type TokenContent interface { + core.AstNode + + IsTokenContent() +} + +func NewTokenContent() TokenContent { + return &TokenContentImpl{} +} + +type TokenContentData struct { +} + +func (i *TokenContentData) IsTokenContent() {} + +func (i *TokenContentData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *TokenContentData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +type TokenContentImpl struct { + core.AstNodeBase + TokenContentData +} + +func (i *TokenContentImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenContentData.ForEachNode(fn) +} + +func (i *TokenContentImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenContentData.ForEachReference(fn) +} + +func (i *TokenContentImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenContentImpl.Resolve: field '%s' does not exist in node '%s' of type 'TokenContent'", field.Value(), nodePath) +} + +type RegexpTokenContent interface { + core.AstNode + TokenContent + + IsRegexpTokenContent() Regexp() string RegexpToken() *core.Token SetRegexp(value *core.Token) } -func NewToken() Token { - return &TokenImpl{} +func NewRegexpTokenContent() RegexpTokenContent { + return &RegexpTokenContentImpl{} } -type TokenData struct { - _Type *core.Token +type RegexpTokenContentData struct { regexp *core.Token } -func (i *TokenData) IsToken() {} +func (i *RegexpTokenContentData) IsRegexpTokenContent() {} + +func (i *RegexpTokenContentData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *RegexpTokenContentData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *RegexpTokenContentData) Regexp() string { + if i != nil && i.regexp != nil { + return i.regexp.Image + } else { + return "" + } +} + +func (i *RegexpTokenContentData) RegexpToken() *core.Token { + return i.regexp +} + +func (i *RegexpTokenContentData) SetRegexp(value *core.Token) { + i.regexp = value +} + +type RegexpTokenContentImpl struct { + core.AstNodeBase + TokenContentData + RegexpTokenContentData +} + +func (i *RegexpTokenContentImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenContentData.ForEachNode(fn) + i.RegexpTokenContentData.ForEachNode(fn) +} + +func (i *RegexpTokenContentImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenContentData.ForEachReference(fn) + i.RegexpTokenContentData.ForEachReference(fn) +} + +func (i *RegexpTokenContentImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameRegexp: + return nil, fmt.Errorf("RegexpTokenContentImpl.Resolve: field 'regexp' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("RegexpTokenContentImpl.Resolve: field '%s' does not exist in node '%s' of type 'RegexpTokenContent'", field.Value(), nodePath) + } +} + +type KeywordTokenContent interface { + core.AstNode + TokenContent + + IsKeywordTokenContent() + Keyword() Keyword + SetKeyword(value Keyword) +} + +func NewKeywordTokenContent() KeywordTokenContent { + return &KeywordTokenContentImpl{} +} + +type KeywordTokenContentData struct { + keyword Keyword +} + +func (i *KeywordTokenContentData) IsKeywordTokenContent() {} + +func (i *KeywordTokenContentData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.keyword != nil { + fn(i.keyword, fieldNameKeyword, -1) + } +} + +func (i *KeywordTokenContentData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *KeywordTokenContentData) Keyword() Keyword { + if i != nil && i.keyword != nil { + return i.keyword + } else { + return nil + } +} + +func (i *KeywordTokenContentData) SetKeyword(value Keyword) { + i.keyword = value +} + +type KeywordTokenContentImpl struct { + core.AstNodeBase + TokenContentData + KeywordTokenContentData +} + +func (i *KeywordTokenContentImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenContentData.ForEachNode(fn) + i.KeywordTokenContentData.ForEachNode(fn) +} + +func (i *KeywordTokenContentImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenContentData.ForEachReference(fn) + i.KeywordTokenContentData.ForEachReference(fn) +} + +func (i *KeywordTokenContentImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameKeyword: + if i.Keyword() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("KeywordTokenContentImpl.Resolve: field 'keyword' is nil in node '%s'", nodePath) + } + child := i.Keyword() + return child.Resolve(path.Tail()) + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("KeywordTokenContentImpl.Resolve: field '%s' does not exist in node '%s' of type 'KeywordTokenContent'", field.Value(), nodePath) + } +} + +type TokenCommand interface { + core.AstNode + + IsTokenCommand() + Type() string + TypeToken() *core.Token + SetType(value *core.Token) + Mode() *core.Reference[TokenMode] + SetMode(value *core.Reference[TokenMode]) + IsDefault() bool + DefaultToken() *core.Token + SetDefault(value *core.Token) +} + +func NewTokenCommand() TokenCommand { + return &TokenCommandImpl{} +} + +type TokenCommandData struct { + _Type *core.Token + mode *core.Reference[TokenMode] + _Default *core.Token +} + +func (i *TokenCommandData) IsTokenCommand() {} -func (i *TokenData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +func (i *TokenCommandData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { } -func (i *TokenData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +func (i *TokenCommandData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + if i.mode != nil { + fn(i.mode, fieldNameMode, -1) + } } -func (i *TokenData) Type() string { +func (i *TokenCommandData) Type() string { if i != nil && i._Type != nil { return i._Type.Image } else { @@ -1069,64 +1425,66 @@ func (i *TokenData) Type() string { } } -func (i *TokenData) TypeToken() *core.Token { +func (i *TokenCommandData) TypeToken() *core.Token { return i._Type } -func (i *TokenData) SetType(value *core.Token) { +func (i *TokenCommandData) SetType(value *core.Token) { i._Type = value } -func (i *TokenData) Regexp() string { - if i != nil && i.regexp != nil { - return i.regexp.Image +func (i *TokenCommandData) Mode() *core.Reference[TokenMode] { + if i != nil && i.mode != nil { + return i.mode } else { - return "" + return nil } } -func (i *TokenData) RegexpToken() *core.Token { - return i.regexp +func (i *TokenCommandData) SetMode(value *core.Reference[TokenMode]) { + i.mode = value } -func (i *TokenData) SetRegexp(value *core.Token) { - i.regexp = value +func (i *TokenCommandData) IsDefault() bool { + return i != nil && i._Default != nil +} + +func (i *TokenCommandData) DefaultToken() *core.Token { + return i._Default +} + +func (i *TokenCommandData) SetDefault(value *core.Token) { + i._Default = value } -type TokenImpl struct { +type TokenCommandImpl struct { core.AstNodeBase - AbstractTokenRuleData - AbstractRuleData - TokenData + TokenCommandData } -func (i *TokenImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { - i.AbstractTokenRuleData.ForEachNode(fn) - i.AbstractRuleData.ForEachNode(fn) - i.TokenData.ForEachNode(fn) +func (i *TokenCommandImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenCommandData.ForEachNode(fn) } -func (i *TokenImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { - i.AbstractTokenRuleData.ForEachReference(fn) - i.AbstractRuleData.ForEachReference(fn) - i.TokenData.ForEachReference(fn) +func (i *TokenCommandImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenCommandData.ForEachReference(fn) } -func (i *TokenImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { +func (i *TokenCommandImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { if path.Empty() { return i, nil } field, _ := path.Head() switch field { - case fieldNameName: - return nil, fmt.Errorf("TokenImpl.Resolve: field 'name' holds a primitive value instead of an ast node") - case fieldNameRegexp: - return nil, fmt.Errorf("TokenImpl.Resolve: field 'regexp' holds a primitive value instead of an ast node") + case fieldNameDefault: + return nil, fmt.Errorf("TokenCommandImpl.Resolve: field '_Default' holds a primitive value instead of an ast node") case fieldNameType: - return nil, fmt.Errorf("TokenImpl.Resolve: field '_Type' holds a primitive value instead of an ast node") + return nil, fmt.Errorf("TokenCommandImpl.Resolve: field '_Type' holds a primitive value instead of an ast node") + case fieldNameMode: + return nil, fmt.Errorf("TokenCommandImpl.Resolve: field 'mode' is a cross-reference instead of a container field") default: nodePath, _ := core.PathOf(i) - return nil, fmt.Errorf("TokenImpl.Resolve: field '%s' does not exist in node '%s' of type 'Token'", field.Value(), nodePath) + return nil, fmt.Errorf("TokenCommandImpl.Resolve: field '%s' does not exist in node '%s' of type 'TokenCommand'", field.Value(), nodePath) } } @@ -1137,10 +1495,10 @@ type TokenGroup interface { IsTokenGroup() TokenRefs() []*core.Reference[AbstractTokenRule] SetTokenRefsItem(item *core.Reference[AbstractTokenRule]) - Regexps() []*core.Token - SetRegexpsItem(item *core.Token) Keywords() []Keyword SetKeywordsItem(item Keyword) + KeywordSelectors() []*core.Token + SetKeywordSelectorsItem(item *core.Token) } func NewTokenGroup() TokenGroup { @@ -1148,9 +1506,9 @@ func NewTokenGroup() TokenGroup { } type TokenGroupData struct { - tokenRefs []*core.Reference[AbstractTokenRule] - regexps []*core.Token - keywords []Keyword + tokenRefs []*core.Reference[AbstractTokenRule] + keywords []Keyword + keywordSelectors []*core.Token } func (i *TokenGroupData) IsTokenGroup() {} @@ -1175,14 +1533,6 @@ func (i *TokenGroupData) SetTokenRefsItem(item *core.Reference[AbstractTokenRule i.tokenRefs = append(i.tokenRefs, item) } -func (i *TokenGroupData) Regexps() []*core.Token { - return i.regexps -} - -func (i *TokenGroupData) SetRegexpsItem(item *core.Token) { - i.regexps = append(i.regexps, item) -} - func (i *TokenGroupData) Keywords() []Keyword { return i.keywords } @@ -1191,6 +1541,14 @@ func (i *TokenGroupData) SetKeywordsItem(item Keyword) { i.keywords = append(i.keywords, item) } +func (i *TokenGroupData) KeywordSelectors() []*core.Token { + return i.keywordSelectors +} + +func (i *TokenGroupData) SetKeywordSelectorsItem(item *core.Token) { + i.keywordSelectors = append(i.keywordSelectors, item) +} + type TokenGroupImpl struct { core.AstNodeBase AbstractTokenRuleData @@ -1216,6 +1574,13 @@ func (i *TokenGroupImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { } field, index := path.Head() switch field { + case fieldNameCommand: + if i.Command() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenGroupImpl.Resolve: field 'command' is nil in node '%s'", nodePath) + } + child := i.Command() + return child.Resolve(path.Tail()) case fieldNameKeywords: if index >= len(i.Keywords()) { nodePath, _ := core.PathOf(i) @@ -1227,10 +1592,12 @@ func (i *TokenGroupImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { return nil, fmt.Errorf("TokenGroupImpl.Resolve: item %d of slice in field 'keywords' is nil in node '%s'", index, nodePath) } return child.Resolve(path.Tail()) + case fieldNameKeywordSelectors: + return nil, fmt.Errorf("TokenGroupImpl.Resolve: field 'keywordSelectors' holds a primitive value instead of an ast node") + case fieldNameModifier: + return nil, fmt.Errorf("TokenGroupImpl.Resolve: field 'modifier' holds a primitive value instead of an ast node") case fieldNameName: return nil, fmt.Errorf("TokenGroupImpl.Resolve: field 'name' holds a primitive value instead of an ast node") - case fieldNameRegexps: - return nil, fmt.Errorf("TokenGroupImpl.Resolve: field 'regexps' holds a primitive value instead of an ast node") case fieldNameTokenRefs: return nil, fmt.Errorf("TokenGroupImpl.Resolve: field 'tokenRefs' is a cross-reference instead of a container field") default: @@ -1239,6 +1606,621 @@ func (i *TokenGroupImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { } } +type TokenMode interface { + core.AstNode + + IsTokenMode() + Name() string + NameToken() *core.Token + SetName(value *core.Token) + IsDefault() bool + DefaultToken() *core.Token + SetDefault(value *core.Token) + Members() []TokenModeMember + SetMembersItem(item TokenModeMember) +} + +func NewTokenMode() TokenMode { + return &TokenModeImpl{} +} + +type TokenModeData struct { + name *core.Token + _Default *core.Token + members []TokenModeMember +} + +func (i *TokenModeData) IsTokenMode() {} + +func (i *TokenModeData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + for j, item := range i.members { + fn(item, fieldNameMembers, j) + } +} + +func (i *TokenModeData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *TokenModeData) Name() string { + if i != nil && i.name != nil { + return i.name.Image + } else { + return "" + } +} + +func (i *TokenModeData) NameToken() *core.Token { + return i.name +} + +func (i *TokenModeData) SetName(value *core.Token) { + i.name = value +} + +func (i *TokenModeData) IsDefault() bool { + return i != nil && i._Default != nil +} + +func (i *TokenModeData) DefaultToken() *core.Token { + return i._Default +} + +func (i *TokenModeData) SetDefault(value *core.Token) { + i._Default = value +} + +func (i *TokenModeData) Members() []TokenModeMember { + return i.members +} + +func (i *TokenModeData) SetMembersItem(item TokenModeMember) { + i.members = append(i.members, item) +} + +type TokenModeImpl struct { + core.AstNodeBase + TokenModeData +} + +func (i *TokenModeImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenModeData.ForEachNode(fn) +} + +func (i *TokenModeImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenModeData.ForEachReference(fn) +} + +func (i *TokenModeImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, index := path.Head() + switch field { + case fieldNameMembers: + if index >= len(i.Members()) { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenModeImpl.Resolve: index %d exceeds length of slice in 'members' (length=%d) in node '%s'", index, len(i.Members()), nodePath) + } + child := i.Members()[index] + if child == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenModeImpl.Resolve: item %d of slice in field 'members' is nil in node '%s'", index, nodePath) + } + return child.Resolve(path.Tail()) + case fieldNameDefault: + return nil, fmt.Errorf("TokenModeImpl.Resolve: field '_Default' holds a primitive value instead of an ast node") + case fieldNameName: + return nil, fmt.Errorf("TokenModeImpl.Resolve: field 'name' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenModeImpl.Resolve: field '%s' does not exist in node '%s' of type 'TokenMode'", field.Value(), nodePath) + } +} + +type TokenModeMember interface { + core.AstNode + + IsTokenModeMember() +} + +func NewTokenModeMember() TokenModeMember { + return &TokenModeMemberImpl{} +} + +type TokenModeMemberData struct { +} + +func (i *TokenModeMemberData) IsTokenModeMember() {} + +func (i *TokenModeMemberData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *TokenModeMemberData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +type TokenModeMemberImpl struct { + core.AstNodeBase + TokenModeMemberData +} + +func (i *TokenModeMemberImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachNode(fn) +} + +func (i *TokenModeMemberImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachReference(fn) +} + +func (i *TokenModeMemberImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenModeMemberImpl.Resolve: field '%s' does not exist in node '%s' of type 'TokenModeMember'", field.Value(), nodePath) +} + +type TokenDeclUsage interface { + core.AstNode + TokenModeMember + + IsTokenDeclUsage() + Declaration() TokenDecl + SetDeclaration(value TokenDecl) +} + +func NewTokenDeclUsage() TokenDeclUsage { + return &TokenDeclUsageImpl{} +} + +type TokenDeclUsageData struct { + declaration TokenDecl +} + +func (i *TokenDeclUsageData) IsTokenDeclUsage() {} + +func (i *TokenDeclUsageData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.declaration != nil { + fn(i.declaration, fieldNameDeclaration, -1) + } +} + +func (i *TokenDeclUsageData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *TokenDeclUsageData) Declaration() TokenDecl { + if i != nil && i.declaration != nil { + return i.declaration + } else { + return nil + } +} + +func (i *TokenDeclUsageData) SetDeclaration(value TokenDecl) { + i.declaration = value +} + +type TokenDeclUsageImpl struct { + core.AstNodeBase + TokenModeMemberData + TokenDeclUsageData +} + +func (i *TokenDeclUsageImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachNode(fn) + i.TokenDeclUsageData.ForEachNode(fn) +} + +func (i *TokenDeclUsageImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachReference(fn) + i.TokenDeclUsageData.ForEachReference(fn) +} + +func (i *TokenDeclUsageImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameDeclaration: + if i.Declaration() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenDeclUsageImpl.Resolve: field 'declaration' is nil in node '%s'", nodePath) + } + child := i.Declaration() + return child.Resolve(path.Tail()) + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenDeclUsageImpl.Resolve: field '%s' does not exist in node '%s' of type 'TokenDeclUsage'", field.Value(), nodePath) + } +} + +type TokenGroupUsage interface { + core.AstNode + TokenModeMember + + IsTokenGroupUsage() + Group() TokenGroup + SetGroup(value TokenGroup) +} + +func NewTokenGroupUsage() TokenGroupUsage { + return &TokenGroupUsageImpl{} +} + +type TokenGroupUsageData struct { + group TokenGroup +} + +func (i *TokenGroupUsageData) IsTokenGroupUsage() {} + +func (i *TokenGroupUsageData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.group != nil { + fn(i.group, fieldNameGroup, -1) + } +} + +func (i *TokenGroupUsageData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *TokenGroupUsageData) Group() TokenGroup { + if i != nil && i.group != nil { + return i.group + } else { + return nil + } +} + +func (i *TokenGroupUsageData) SetGroup(value TokenGroup) { + i.group = value +} + +type TokenGroupUsageImpl struct { + core.AstNodeBase + TokenModeMemberData + TokenGroupUsageData +} + +func (i *TokenGroupUsageImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachNode(fn) + i.TokenGroupUsageData.ForEachNode(fn) +} + +func (i *TokenGroupUsageImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachReference(fn) + i.TokenGroupUsageData.ForEachReference(fn) +} + +func (i *TokenGroupUsageImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameGroup: + if i.Group() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenGroupUsageImpl.Resolve: field 'group' is nil in node '%s'", nodePath) + } + child := i.Group() + return child.Resolve(path.Tail()) + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenGroupUsageImpl.Resolve: field '%s' does not exist in node '%s' of type 'TokenGroupUsage'", field.Value(), nodePath) + } +} + +type TokenUsage interface { + core.AstNode + TokenModeMember + + IsTokenUsage() + Modifier() string + ModifierToken() *core.Token + SetModifier(value *core.Token) + TokenRef() *core.Reference[AbstractTokenRule] + SetTokenRef(value *core.Reference[AbstractTokenRule]) + Command() TokenCommand + SetCommand(value TokenCommand) +} + +func NewTokenUsage() TokenUsage { + return &TokenUsageImpl{} +} + +type TokenUsageData struct { + modifier *core.Token + tokenRef *core.Reference[AbstractTokenRule] + command TokenCommand +} + +func (i *TokenUsageData) IsTokenUsage() {} + +func (i *TokenUsageData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.command != nil { + fn(i.command, fieldNameCommand, -1) + } +} + +func (i *TokenUsageData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + if i.tokenRef != nil { + fn(i.tokenRef, fieldNameTokenRef, -1) + } +} + +func (i *TokenUsageData) Modifier() string { + if i != nil && i.modifier != nil { + return i.modifier.Image + } else { + return "" + } +} + +func (i *TokenUsageData) ModifierToken() *core.Token { + return i.modifier +} + +func (i *TokenUsageData) SetModifier(value *core.Token) { + i.modifier = value +} + +func (i *TokenUsageData) TokenRef() *core.Reference[AbstractTokenRule] { + if i != nil && i.tokenRef != nil { + return i.tokenRef + } else { + return nil + } +} + +func (i *TokenUsageData) SetTokenRef(value *core.Reference[AbstractTokenRule]) { + i.tokenRef = value +} + +func (i *TokenUsageData) Command() TokenCommand { + if i != nil && i.command != nil { + return i.command + } else { + return nil + } +} + +func (i *TokenUsageData) SetCommand(value TokenCommand) { + i.command = value +} + +type TokenUsageImpl struct { + core.AstNodeBase + TokenModeMemberData + TokenUsageData +} + +func (i *TokenUsageImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachNode(fn) + i.TokenUsageData.ForEachNode(fn) +} + +func (i *TokenUsageImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachReference(fn) + i.TokenUsageData.ForEachReference(fn) +} + +func (i *TokenUsageImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameCommand: + if i.Command() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenUsageImpl.Resolve: field 'command' is nil in node '%s'", nodePath) + } + child := i.Command() + return child.Resolve(path.Tail()) + case fieldNameModifier: + return nil, fmt.Errorf("TokenUsageImpl.Resolve: field 'modifier' holds a primitive value instead of an ast node") + case fieldNameTokenRef: + return nil, fmt.Errorf("TokenUsageImpl.Resolve: field 'tokenRef' is a cross-reference instead of a container field") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("TokenUsageImpl.Resolve: field '%s' does not exist in node '%s' of type 'TokenUsage'", field.Value(), nodePath) + } +} + +type KeywordUsage interface { + core.AstNode + TokenModeMember + + IsKeywordUsage() + Modifier() string + ModifierToken() *core.Token + SetModifier(value *core.Token) + Keyword() Keyword + SetKeyword(value Keyword) + Command() TokenCommand + SetCommand(value TokenCommand) +} + +func NewKeywordUsage() KeywordUsage { + return &KeywordUsageImpl{} +} + +type KeywordUsageData struct { + modifier *core.Token + keyword Keyword + command TokenCommand +} + +func (i *KeywordUsageData) IsKeywordUsage() {} + +func (i *KeywordUsageData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.keyword != nil { + fn(i.keyword, fieldNameKeyword, -1) + } + if i.command != nil { + fn(i.command, fieldNameCommand, -1) + } +} + +func (i *KeywordUsageData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *KeywordUsageData) Modifier() string { + if i != nil && i.modifier != nil { + return i.modifier.Image + } else { + return "" + } +} + +func (i *KeywordUsageData) ModifierToken() *core.Token { + return i.modifier +} + +func (i *KeywordUsageData) SetModifier(value *core.Token) { + i.modifier = value +} + +func (i *KeywordUsageData) Keyword() Keyword { + if i != nil && i.keyword != nil { + return i.keyword + } else { + return nil + } +} + +func (i *KeywordUsageData) SetKeyword(value Keyword) { + i.keyword = value +} + +func (i *KeywordUsageData) Command() TokenCommand { + if i != nil && i.command != nil { + return i.command + } else { + return nil + } +} + +func (i *KeywordUsageData) SetCommand(value TokenCommand) { + i.command = value +} + +type KeywordUsageImpl struct { + core.AstNodeBase + TokenModeMemberData + KeywordUsageData +} + +func (i *KeywordUsageImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachNode(fn) + i.KeywordUsageData.ForEachNode(fn) +} + +func (i *KeywordUsageImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachReference(fn) + i.KeywordUsageData.ForEachReference(fn) +} + +func (i *KeywordUsageImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameCommand: + if i.Command() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("KeywordUsageImpl.Resolve: field 'command' is nil in node '%s'", nodePath) + } + child := i.Command() + return child.Resolve(path.Tail()) + case fieldNameKeyword: + if i.Keyword() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("KeywordUsageImpl.Resolve: field 'keyword' is nil in node '%s'", nodePath) + } + child := i.Keyword() + return child.Resolve(path.Tail()) + case fieldNameModifier: + return nil, fmt.Errorf("KeywordUsageImpl.Resolve: field 'modifier' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("KeywordUsageImpl.Resolve: field '%s' does not exist in node '%s' of type 'KeywordUsage'", field.Value(), nodePath) + } +} + +type KeywordSelector interface { + core.AstNode + TokenModeMember + + IsKeywordSelector() + Selector() string + SelectorToken() *core.Token + SetSelector(value *core.Token) +} + +func NewKeywordSelector() KeywordSelector { + return &KeywordSelectorImpl{} +} + +type KeywordSelectorData struct { + selector *core.Token +} + +func (i *KeywordSelectorData) IsKeywordSelector() {} + +func (i *KeywordSelectorData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *KeywordSelectorData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *KeywordSelectorData) Selector() string { + if i != nil && i.selector != nil { + return i.selector.Image + } else { + return "" + } +} + +func (i *KeywordSelectorData) SelectorToken() *core.Token { + return i.selector +} + +func (i *KeywordSelectorData) SetSelector(value *core.Token) { + i.selector = value +} + +type KeywordSelectorImpl struct { + core.AstNodeBase + TokenModeMemberData + KeywordSelectorData +} + +func (i *KeywordSelectorImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachNode(fn) + i.KeywordSelectorData.ForEachNode(fn) +} + +func (i *KeywordSelectorImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.TokenModeMemberData.ForEachReference(fn) + i.KeywordSelectorData.ForEachReference(fn) +} + +func (i *KeywordSelectorImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameSelector: + return nil, fmt.Errorf("KeywordSelectorImpl.Resolve: field 'selector' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("KeywordSelectorImpl.Resolve: field '%s' does not exist in node '%s' of type 'KeywordSelector'", field.Value(), nodePath) + } +} + type Element interface { core.AstNode @@ -2064,30 +3046,42 @@ func (i *CompositeRuleImpl) Resolve(path core.FragmentPath) (core.AstNode, error } var ( - fieldNameType = unique.Make("_Type") - fieldNameAlts = unique.Make("alts") - fieldNameBody = unique.Make("body") - fieldNameCardinality = unique.Make("cardinality") - fieldNameComposites = unique.Make("composites") - fieldNameElements = unique.Make("elements") - fieldNameEntry = unique.Make("entry") - fieldNameExtends = unique.Make("extends") - fieldNameFields = unique.Make("fields") - fieldNameInterfaces = unique.Make("interfaces") - fieldNameInternalType = unique.Make("internalType") - fieldNameKeywords = unique.Make("keywords") - fieldNameName = unique.Make("name") - fieldNameOperator = unique.Make("operator") - fieldNameProperty = unique.Make("property") - fieldNameRegexp = unique.Make("regexp") - fieldNameRegexps = unique.Make("regexps") - fieldNameReturnType = unique.Make("returnType") - fieldNameRule = unique.Make("rule") - fieldNameRules = unique.Make("rules") - fieldNameTerminals = unique.Make("terminals") - fieldNameTokenGroups = unique.Make("tokenGroups") - fieldNameTokenRefs = unique.Make("tokenRefs") - fieldNameValue = unique.Make("value") + fieldNameDefault = unique.Make("_Default") + fieldNameType = unique.Make("_Type") + fieldNameAlts = unique.Make("alts") + fieldNameBody = unique.Make("body") + fieldNameCardinality = unique.Make("cardinality") + fieldNameCommand = unique.Make("command") + fieldNameComposites = unique.Make("composites") + fieldNameContent = unique.Make("content") + fieldNameDeclaration = unique.Make("declaration") + fieldNameElements = unique.Make("elements") + fieldNameEntry = unique.Make("entry") + fieldNameExtends = unique.Make("extends") + fieldNameFields = unique.Make("fields") + fieldNameGroup = unique.Make("group") + fieldNameInterfaces = unique.Make("interfaces") + fieldNameInternalType = unique.Make("internalType") + fieldNameKeyword = unique.Make("keyword") + fieldNameKeywordSelectors = unique.Make("keywordSelectors") + fieldNameKeywords = unique.Make("keywords") + fieldNameMembers = unique.Make("members") + fieldNameMode = unique.Make("mode") + fieldNameModifier = unique.Make("modifier") + fieldNameName = unique.Make("name") + fieldNameOperator = unique.Make("operator") + fieldNameProperty = unique.Make("property") + fieldNameRegexp = unique.Make("regexp") + fieldNameReturnType = unique.Make("returnType") + fieldNameRule = unique.Make("rule") + fieldNameRules = unique.Make("rules") + fieldNameSelector = unique.Make("selector") + fieldNameTerminals = unique.Make("terminals") + fieldNameTokenGroups = unique.Make("tokenGroups") + fieldNameTokenModes = unique.Make("tokenModes") + fieldNameTokenRef = unique.Make("tokenRef") + fieldNameTokenRefs = unique.Make("tokenRefs") + fieldNameValue = unique.Make("value") ) var FastbeltSyntheticFactories = map[string]func() core.AstNode{ @@ -2108,11 +3102,22 @@ var FastbeltSyntheticFactories = map[string]func() core.AstNode{ "Group": func() core.AstNode { return NewGroup() }, "Interface": func() core.AstNode { return NewInterface() }, "Keyword": func() core.AstNode { return NewKeyword() }, + "KeywordSelector": func() core.AstNode { return NewKeywordSelector() }, + "KeywordTokenContent": func() core.AstNode { return NewKeywordTokenContent() }, + "KeywordUsage": func() core.AstNode { return NewKeywordUsage() }, "ParserRule": func() core.AstNode { return NewParserRule() }, "PrimitiveType": func() core.AstNode { return NewPrimitiveType() }, "ReferenceType": func() core.AstNode { return NewReferenceType() }, + "RegexpTokenContent": func() core.AstNode { return NewRegexpTokenContent() }, "RuleCall": func() core.AstNode { return NewRuleCall() }, "SimpleType": func() core.AstNode { return NewSimpleType() }, - "Token": func() core.AstNode { return NewToken() }, + "TokenCommand": func() core.AstNode { return NewTokenCommand() }, + "TokenContent": func() core.AstNode { return NewTokenContent() }, + "TokenDecl": func() core.AstNode { return NewTokenDecl() }, + "TokenDeclUsage": func() core.AstNode { return NewTokenDeclUsage() }, "TokenGroup": func() core.AstNode { return NewTokenGroup() }, + "TokenGroupUsage": func() core.AstNode { return NewTokenGroupUsage() }, + "TokenMode": func() core.AstNode { return NewTokenMode() }, + "TokenModeMember": func() core.AstNode { return NewTokenModeMember() }, + "TokenUsage": func() core.AstNode { return NewTokenUsage() }, } diff --git a/internal/grammar/utils.go b/internal/grammar/utils.go index 1820fca2..eb355616 100644 --- a/internal/grammar/utils.go +++ b/internal/grammar/utils.go @@ -23,7 +23,7 @@ func convertString(keyword Keyword) (string, error) { return "", errMissingKeywordValue } if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' { - value = value[1 : len(value)-1] + value = KeywordValue(keyword) } return value, nil } diff --git a/internal/grammar/validator.go b/internal/grammar/validator.go index 10c5e7d7..93af872e 100644 --- a/internal/grammar/validator.go +++ b/internal/grammar/validator.go @@ -16,29 +16,46 @@ import ( ) const ( - ValidateUniqueRuleName = "uniqueRuleName" - ValidateUniqueInterfaceName = "uniqueInterfaceName" - ValidateEmptyToken = "emptyTerminalRule" - ValidateEmptyKeyword = "emptyKeyword" - ValidateWhitespaceOnlyKeyword = "whitespaceOnlyKeyword" - ValidateKeywordWithWhitespace = "keywordWithWhitespace" - ValidateRuleReturnType = "ruleReturnType" - ValidateInterfaceExtends = "interfaceExtends" - ValidateRuleCallReturnType = "ruleCallReturnType" - ValidateRuleCallPosition = "ruleCallPosition" - ValidateActionAssignmentType = "actionAssignmentType" - ValidateActionPropertyType = "actionPropertyType" - ValidateAssignmentType = "assignmentType" - ValidateRecursiveTokenGroup = "recursiveTokenGroup" - ValidateInvalidTokenInGroup = "invalidTokenInGroup" - ValidateInvalidTokenInCrossRef = "invalidTokenInCrossRef" - ValidateMissingCrossRefTerminal = "missingCrossRefTerminal" - ValidateUniqueFieldName = "uniqueFieldName" - ValidateFieldNameCapitalLetter = "fieldNameCapitalLetter" - ValidateReservedFieldName = "reservedFieldName" - ValidateNestedArrayType = "nestedArrayType" + ValidateUniqueRuleName = "uniqueRuleName" + ValidateUniqueRuleNameInTokenMode = "uniqueRuleNameInTokenMode" + ValidateUniqueInterfaceName = "uniqueInterfaceName" + ValidateUniqueTokenModeName = "uniqueTokenModeName" + ValidateEmptyToken = "emptyTerminalRule" + ValidateEmptyKeyword = "emptyKeyword" + ValidateWhitespaceOnlyKeyword = "whitespaceOnlyKeyword" + ValidateKeywordWithWhitespace = "keywordWithWhitespace" + ValidateRuleReturnType = "ruleReturnType" + ValidateInterfaceExtends = "interfaceExtends" + ValidateRuleCallReturnType = "ruleCallReturnType" + ValidateRuleCallPosition = "ruleCallPosition" + ValidateActionAssignmentType = "actionAssignmentType" + ValidateActionPropertyType = "actionPropertyType" + ValidateAssignmentType = "assignmentType" + ValidateRecursiveTokenGroup = "recursiveTokenGroup" + ValidateInvalidTokenInGroup = "invalidTokenInGroup" + ValidateInvalidTokenInCrossRef = "invalidTokenInCrossRef" + ValidateMissingCrossRefTerminal = "missingCrossRefTerminal" + ValidateUniqueFieldName = "uniqueFieldName" + ValidateFieldNameCapitalLetter = "fieldNameCapitalLetter" + ValidateReservedFieldName = "reservedFieldName" + ValidateNestedArrayType = "nestedArrayType" + ValidateDefaultTokenModeRequired = "defaultTokenModeRequired" + ValidateTokenCommandMode = "tokenCommandMode" + ValidateEmptyTokenMode = "emptyTokenMode" + ValidateUnreachableTokenMode = "unreachableTokenMode" + ValidateKeywordNotInTokenMode = "keywordNotInTokenMode" + ValidateTokenNotInTokenMode = "tokenNotInTokenMode" + ValidateNonDefaultTokenModeNoPop = "nonDefaultTokenModeNoPop" + ValidateTerminalNotCoveredByParserRule = "terminalNotCoveredByParserRule" + ValidateTokenGroupNotCoveredByParserRule = "tokenGroupNotCoveredByParserRule" + ValidateInvalidRegExpLiteral = "invalidRegExpLiteral" + ValidateKeywordPureStandaloneOrTokenDecl = "keywordPureStandaloneOrTokenDecl" ) +// defaultTokenModeName is the name under which the mode marked +// `token mode default` is registered. It is the mode the lexer starts in. +const defaultTokenModeName = "default" + // reservedFieldNames lists field names that must not be used because they // conflict with [core.AstNode] methods. Keep in sync with ast.go. var reservedFieldNames = map[string]string{ @@ -53,12 +70,597 @@ var reservedFieldNames = map[string]string{ "Resolve": "AstNode.Resolve", } -// GrammarImpl.Validate checks grammar-level constraints: -// - Rule names must be unique within the grammar. -// - Interface names must be unique within the grammar. -func (g *GrammarImpl) Validate(_ context.Context, _ string, accept core.ValidationAcceptor) { +// GrammarImpl.Validate checks grammar-level constraints +func (g *GrammarImpl) Validate(ctx context.Context, _ string, accept core.ValidationAcceptor) { checkUniqueRuleNames(g, accept) checkUniqueInterfaceNames(g, accept) + checkUniqueTokenModeNames(g, accept) + checkIfDefaultTokenModeIsRequired(g, accept) + checkTokenModesAreReachable(g, ctx, accept) + checkTokenModesCoverParserTokens(g, ctx, accept) + checkParserRulesCoverVisibleTokens(g, ctx, accept) + checkIfNonDefaultTokenModesHasNoExit(g, ctx, accept) + checkIfKeywordPureStandaloneOrTokenDecl(g, ctx, accept) +} + +// tokenModeName returns the name a token mode is registered under. The mode +// marked `default` has no name of its own. +func tokenModeName(mode TokenMode) string { + if mode.IsDefault() { + return defaultTokenModeName + } + return mode.Name() +} + +// tokenModeNameToken returns the token to anchor diagnostics about mode itself +// on, which is either its name or the `default` marker. +func tokenModeNameToken(mode TokenMode) *core.Token { + if mode.IsDefault() { + return mode.DefaultToken() + } + return mode.NameToken() +} + +// checkTokenModesAreReachable reports token modes that no command switches to. +// The lexer starts in the default mode and can only leave it through a `push` or +// `mode` command, so a mode nothing targets is dead weight. +func checkTokenModesAreReachable(g Grammar, ctx context.Context, accept core.ValidationAcceptor) { + if len(g.TokenModes()) == 0 { + return + } + var entryMode TokenMode = nil + transitions := map[TokenMode][]TokenMode{} + for _, mode := range g.TokenModes() { + if mode.IsDefault() { + entryMode = mode + } + for _, member := range mode.Members() { + command := getCommand(member) + if command != nil && command.Mode() != nil { + transitions[mode] = append(transitions[mode], command.Mode().Ref(ctx)) + } + } + } + visited := collections.NewSet[TokenMode]() + queue := []TokenMode{} + if entryMode != nil { + queue = append(queue, entryMode) + } + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + visited.Add(current) + for _, next := range transitions[current] { + if !visited.Has(next) { + queue = append(queue, next) + } + } + } + for _, mode := range g.TokenModes() { + if visited.Has(mode) { + continue + } + accept(core.NewDiagnostic( + core.SeverityWarning, + fmt.Sprintf("The token mode '%s' is never entered from the default mode. Add a 'push(%s)' or 'mode(%s)' command to a token.", mode.Name(), mode.Name(), mode.Name()), + mode, + core.WithToken(mode.NameToken()), + core.WithCode(ValidateUnreachableTokenMode), + )) + } +} + +func checkIfNonDefaultTokenModesHasNoExit(g Grammar, ctx context.Context, accept core.ValidationAcceptor) { + if len(g.TokenModes()) == 0 { + return + } + for _, mode := range g.TokenModes() { + if mode.IsDefault() { + continue + } + found := false + for _, member := range mode.Members() { + command := getCommand(member) + if command != nil && command.Type() != tokenCommandPush { + found = true + } + } + if !found { + accept(core.NewDiagnostic( + core.SeverityWarning, + fmt.Sprintf("The non-default token mode '%s' has no exit command ('mode' or 'pop').", mode.Name()), + mode, + core.WithToken(mode.NameToken()), + core.WithCode(ValidateNonDefaultTokenModeNoPop), + )) + } + } +} + +func getCommand(member TokenModeMember) TokenCommand { + switch casted := member.(type) { + case TokenDeclUsage: + return casted.Declaration().Command() + case TokenGroupUsage: + return casted.Group().Command() + case TokenUsage: + command := casted.Command() + if command == nil { + ref := casted.TokenRef().Ref(context.Background()) + if ref != nil { + command = ref.Command() + } + } + return command + case KeywordUsage: + return casted.Command() + case KeywordSelector: + return nil + } + return nil +} + +func checkIfDefaultTokenModeIsRequired(g Grammar, accept core.ValidationAcceptor) { + if len(g.TokenModes()) > 0 { + hasDefault := false + var nonDefaultTokenMode TokenMode = nil + for _, mode := range g.TokenModes() { + if mode.IsDefault() { + hasDefault = true + break + } else if nonDefaultTokenMode == nil { + //mark only the first non-default token mode + //one diagnostic is enough to indicate that a default token mode is required + nonDefaultTokenMode = mode + } + } + if !hasDefault { + accept(core.NewDiagnostic( + core.SeverityError, + "At least one token mode must be marked as default.", + nonDefaultTokenMode, + core.WithToken(nonDefaultTokenMode.NameToken()), + core.WithCode(ValidateDefaultTokenModeRequired), + )) + } + } +} + +func checkUniqueTokenModeNames(g Grammar, accept core.ValidationAcceptor) { + seen := map[string][]TokenMode{} + for _, mode := range g.TokenModes() { + name := tokenModeName(mode) + seen[name] = append(seen[name], mode) + } + for name, modes := range seen { + if len(modes) > 1 { + for _, mode := range modes { + token := tokenModeNameToken(mode) + accept(core.NewDiagnostic( + core.SeverityError, + fmt.Sprintf("A token mode's name has to be unique. '%s' is used multiple times.", name), + mode, + core.WithToken(token), + core.WithCode(ValidateUniqueTokenModeName), + )) + } + } + } +} + +// Token command types as they appear after the `->` arrow. +const ( + tokenCommandPush = "push" + tokenCommandPop = "pop" + tokenCommandMode = "mode" +) + +// TokenCommandImpl.Validate checks lexer mode switch constraints: +// - `push` and `mode` need a target mode, otherwise there is nothing to +// switch to and the command is dropped during code generation. +// - `pop` returns to the mode below the current one on the stack, so a target +// mode cannot be honored. +func (c *TokenCommandImpl) Validate(_ context.Context, _ string, accept core.ValidationAcceptor) { + checkTokenCommandMode(c, accept) +} + +func checkTokenCommandMode(c TokenCommand, accept core.ValidationAcceptor) { + // An unresolvable mode reference is still a target; the linker reports it. + hasMode := c.IsDefault() || c.Mode() != nil + switch c.Type() { + case tokenCommandPop: + if !hasMode { + return + } + options := []core.DiagnosticOption{core.WithCode(ValidateTokenCommandMode)} + if c.Mode() != nil { + options = append(options, core.WithReference(c.Mode())) + } else { + options = append(options, core.WithToken(c.DefaultToken())) + } + accept(core.NewDiagnostic( + core.SeverityError, + "The 'pop' command returns to the previous token mode and cannot take a target mode.", + c, + options..., + )) + case tokenCommandPush, tokenCommandMode: + if hasMode { + return + } + accept(core.NewDiagnostic( + core.SeverityError, + fmt.Sprintf("The '%s' command requires a target token mode, for example '%s(default)'.", c.Type(), c.Type()), + c, + core.WithToken(c.TypeToken()), + core.WithCode(ValidateTokenCommandMode), + )) + } +} + +// TokenModeImpl.Validate checks token mode constraints: +// - A mode without members leaves the lexer with nothing to match. +func (m *TokenModeImpl) Validate(_ context.Context, _ string, accept core.ValidationAcceptor) { + checkTokenModeNotEmpty(m, accept) + checkTokenModeMembersAreUnique(m, accept) +} + +func checkTokenModeNotEmpty(mode TokenMode, accept core.ValidationAcceptor) { + if len(mode.Members()) > 0 { + return + } + accept(core.NewDiagnostic( + core.SeverityWarning, + fmt.Sprintf("The token mode '%s' does not declare any token, so the lexer cannot match anything while it is active.", tokenModeName(mode)), + mode, + core.WithToken(tokenModeNameToken(mode)), + core.WithCode(ValidateEmptyTokenMode), + )) +} + +func getTokenNeverReferencedMessage(tokenTypeName string, tokenValue string) string { + return fmt.Sprintf("The %s '%s' is never referenced in a parser rule, so the lexer can never produce it.", tokenTypeName, tokenValue) +} + +func checkParserRulesCoverVisibleTokens(g Grammar, ctx context.Context, accept core.ValidationAcceptor) { + severity := core.SeverityWarning + seen := collections.NewSet[string]() + queue := []core.AstNode{} + for _, composite := range g.Composites() { + for node := range core.AllChildren(composite) { + queue = append(queue, node) + } + } + for _, rule := range g.Rules() { + for node := range core.AllChildren(rule) { + queue = append(queue, node) + } + } + for _, group := range g.TokenGroups() { + for node := range core.AllChildren(group) { + queue = append(queue, node) + } + } + for _, node := range queue { + switch casted := node.(type) { + case Keyword: + seen.Add(casted.Value()) + case RuleCall: + rule, ok := casted.Rule().Ref(ctx).(AbstractTokenRule) + if ok { + seen.Add(rule.Name()) + } + } + } + + if len(g.TokenModes()) == 0 { + for _, terminal := range g.Terminals() { + if terminal.Modifier() != "" { + continue + } + if !seen.Has(terminal.Name()) { + accept(core.NewDiagnostic( + severity, + getTokenNeverReferencedMessage("token", terminal.Name()), + terminal, + core.WithToken(terminal.NameToken()), + core.WithCode(ValidateTerminalNotCoveredByParserRule), + )) + } + } + for _, group := range g.TokenGroups() { + if group.Modifier() != "" { + continue + } + if !seen.Has(group.Name()) { + accept(core.NewDiagnostic( + severity, + getTokenNeverReferencedMessage("token group", group.Name()), + group, + core.WithToken(group.NameToken()), + core.WithCode(ValidateTokenGroupNotCoveredByParserRule), + )) + } + } + } else { + for _, mode := range g.TokenModes() { + for _, member := range mode.Members() { + switch member := member.(type) { + case KeywordUsage: + if !seen.Has(member.Keyword().Value()) { + accept(core.NewDiagnostic( + severity, + getTokenNeverReferencedMessage("keyword", member.Keyword().Value()), + member.Keyword(), + core.WithToken(member.Keyword().ValueToken()), + core.WithCode(ValidateTerminalNotCoveredByParserRule), + )) + } + case TokenUsage: + if tokenRef := member.TokenRef().Ref(context.Background()); tokenRef != nil { + if tokenRef.Modifier() != "" { + continue + } + if !seen.Has(tokenRef.Name()) { + accept(core.NewDiagnostic( + severity, + getTokenNeverReferencedMessage("token", tokenRef.Name()), + member, + core.WithCode(ValidateTerminalNotCoveredByParserRule), + )) + } + } + case TokenDeclUsage: + if member.Declaration().Modifier() != "" { + continue + } + if !seen.Has(member.Declaration().Name()) { + accept(core.NewDiagnostic( + severity, + getTokenNeverReferencedMessage("token", member.Declaration().Name()), + member.Declaration(), + core.WithToken(member.Declaration().NameToken()), + core.WithCode(ValidateTerminalNotCoveredByParserRule), + )) + } + case TokenGroupUsage: + if member.Group().Modifier() != "" { + continue + } + if !seen.Has(member.Group().Name()) { + accept(core.NewDiagnostic( + severity, + getTokenNeverReferencedMessage("token group", member.Group().Name()), + member.Group(), + core.WithToken(member.Group().NameToken()), + core.WithCode(ValidateTokenGroupNotCoveredByParserRule), + )) + } + } + } + } + } +} + +// tokenModeCoverage records the keywords and token rules that the declared token +// modes register with the lexer. +type tokenModeCoverage struct { + keywords collections.Set[string] + rules collections.Set[string] +} + +// checkTokenModesCoverParserTokens reports keywords and tokens that the parser +// can demand but that no token mode registers. Declaring token modes takes over +// token registration from the grammar as a whole, so anything left out of every +// mode can never be produced by the lexer and makes the rule using it dead. +// +// Only the first occurrence of each keyword or token is reported: the fix is a +// single entry in a token mode, not one per use site. +func checkTokenModesCoverParserTokens(g Grammar, ctx context.Context, accept core.ValidationAcceptor) { + if len(g.TokenModes()) == 0 { + // Without explicit token modes every keyword and token is registered + // automatically, so nothing can be missing. + return + } + coverage := collectTokenModeCoverage(g, ctx) + reported := collections.NewSet[string]() +outerLoop: + for node := range core.AllChildren(g) { + switch node := node.(type) { + case Keyword: + value := node.Value() + if !insideParserRule(node) || coverage.keywords.Has(value) || !reported.Add(value) { + continue + } + accept(core.NewDiagnostic( + core.SeverityError, + fmt.Sprintf("The keyword %s is not registered in any token mode, so the lexer can never produce it. List it in a token mode or cover it with a 'keywords' selector.", value), + node, + core.WithToken(node.ValueToken()), + core.WithCode(ValidateKeywordNotInTokenMode), + )) + case RuleCall: + rule, ok := node.Rule().Ref(ctx).(AbstractTokenRule) + if !ok || !insideParserRule(node) || coverage.covers(rule) || !reported.Add(rule.Name()) { + continue + } + if tokenGroup, ok := rule.(TokenGroup); ok { + //Ignore token groups FTM, since they are meant to be + //used as a grouped token type during parsing, not during lexing + //Instead test the individual token types in the group for coverage + //If all token types in the group are not covered, then the individual token group will be reported as missing + for _, member := range tokenGroup.TokenRefs() { + if member := member.Ref(ctx); member != nil && coverage.covers(member) { + continue outerLoop + } + } + for _, member := range tokenGroup.Keywords() { + value := KeywordValue(member) + if coverage.keywords.Has(value) { + continue outerLoop + } + } + for _, member := range tokenGroup.KeywordSelectors() { + pattern, err := regexp.Compile(RegexpValue(member.Image)) + if err != nil { + //error is already reported by the grammar validator, so ignore it here + continue + } + for _, keyword := range allGrammarKeywords(g) { + value := KeywordValue(keyword) + if pattern.MatchString(value) && coverage.keywords.Has(value) { + continue outerLoop + } + } + } + } + // Deliberately a warning, not an error: unlike a keyword, a token + // rule may be registered with the lexer by hand-written code, and a + // grammar that leaves one out still generates and builds. + accept(core.NewDiagnostic( + core.SeverityWarning, + fmt.Sprintf("The token '%s' is not registered in any token mode, so the lexer can never produce it. List it in a token mode.", rule.Name()), + rule, + core.WithToken(rule.NameToken()), + core.WithCode(ValidateTokenNotInTokenMode), + )) + } + } +} + +// insideParserRule reports whether node contributes to what the parser matches. +// Keywords and rule calls nested in a token declaration, token group or token +// mode describe the lexer instead and are therefore not parser input. +func insideParserRule(node core.AstNode) bool { + for container := node.Container(); container != nil; container = container.Container() { + switch container.(type) { + case ParserRule, CompositeRule: + return true + case TokenDecl, TokenGroup, TokenMode: + return false + } + } + return false +} + +func collectTokenModeCoverage(g Grammar, ctx context.Context) tokenModeCoverage { + coverage := tokenModeCoverage{ + keywords: collections.NewSet[string](), + rules: collections.NewSet[string](), + } + allKeywords := allGrammarKeywords(g) + // Shared across modes: coverage is grammar-wide, so a group already expanded + // for one mode needs no second pass, and the set breaks reference cycles. + visitedGroups := collections.NewSet[string]() + for _, mode := range g.TokenModes() { + for _, member := range mode.Members() { + switch member := member.(type) { + case TokenDeclUsage: + coverage.addTokenDecl(member.Declaration()) + case TokenGroupUsage: + coverage.addTokenGroup(member.Group(), ctx, allKeywords, visitedGroups) + case TokenUsage: + coverage.addTokenRule(member.TokenRef().Ref(ctx), ctx, allKeywords, visitedGroups) + case KeywordUsage: + coverage.addKeyword(member.Keyword()) + case KeywordSelector: + coverage.addKeywordsMatching(member.Selector(), allKeywords) + } + } + } + return coverage +} + +// covers reports whether the lexer can produce a token for rule. +func (c tokenModeCoverage) covers(rule AbstractTokenRule) bool { + if c.rules.Has(rule.Name()) { + return true + } + // A token declared as a single keyword shares that keyword's token id, so + // covering the keyword - through a 'keywords' selector, for instance - + // covers the token as well. + if decl, ok := rule.(TokenDecl); ok { + if content, ok := decl.Content().(KeywordTokenContent); ok && content.Keyword() != nil { + return c.keywords.Has(content.Keyword().Value()) + } + } + return false +} + +func (c tokenModeCoverage) addTokenRule(rule AbstractTokenRule, ctx context.Context, allKeywords []Keyword, visited collections.Set[string]) { + switch rule := rule.(type) { + case TokenDecl: + c.addTokenDecl(rule) + case TokenGroup: + c.addTokenGroup(rule, ctx, allKeywords, visited) + } +} + +func (c tokenModeCoverage) addTokenDecl(decl TokenDecl) { + if decl == nil { + return + } + c.rules.Add(decl.Name()) + // `token LBRACE: "{"` registers the keyword under the token's name, so + // listing the token covers the keyword as well. + if content, ok := decl.Content().(KeywordTokenContent); ok { + c.addKeyword(content.Keyword()) + } +} + +func (c tokenModeCoverage) addTokenGroup(group TokenGroup, ctx context.Context, allKeywords []Keyword, visited collections.Set[string]) { + if group == nil || !visited.Add(group.Name()) { + return + } + c.rules.Add(group.Name()) + for _, keyword := range group.Keywords() { + c.addKeyword(keyword) + } + for _, selector := range group.KeywordSelectors() { + c.addKeywordsMatching(selector.Image, allKeywords) + } + for _, tokenRef := range group.TokenRefs() { + c.addTokenRule(tokenRef.Ref(ctx), ctx, allKeywords, visited) + } +} + +func (c tokenModeCoverage) addKeyword(keyword Keyword) { + if keyword == nil || keyword.Value() == "" { + return + } + c.keywords.Add(keyword.Value()) +} + +func (c tokenModeCoverage) addKeywordsMatching(selector string, allKeywords []Keyword) { + if len(selector) < 2 { + return + } + pattern, err := regexp.Compile(RegexpValue(selector)) + if err != nil { + return + } + for _, keyword := range allKeywords { + value, err := convertString(keyword) + if err != nil { + continue + } + if pattern.MatchString(value) { + c.keywords.Add(keyword.Value()) + } + } +} + +// allGrammarKeywords returns every distinct keyword of the grammar, wherever it +// is declared. A `keywords` selector is matched against all of them. +func allGrammarKeywords(g Grammar) []Keyword { + seen := collections.NewSet[string]() + keywords := []Keyword{} + for node := range core.AllChildren(g) { + if keyword, ok := node.(Keyword); ok && seen.Add(keyword.Value()) { + keywords = append(keywords, keyword) + } + } + return keywords } func checkUniqueRuleNames(g Grammar, accept core.ValidationAcceptor) { @@ -78,6 +680,21 @@ func checkUniqueRuleNames(g Grammar, accept core.ValidationAcceptor) { seen[tokenGroup.Name()] = append(seen[tokenGroup.Name()], tokenGroup) } } + for _, tokenMode := range g.TokenModes() { + for _, member := range tokenMode.Members() { + if usage, ok := member.(TokenDeclUsage); ok { + decl := usage.Declaration() + if decl.Name() != "" { + seen[decl.Name()] = append(seen[decl.Name()], decl) + } + } else if group, ok := member.(TokenGroupUsage); ok { + decl := group.Group() + if decl.Name() != "" { + seen[decl.Name()] = append(seen[decl.Name()], decl) + } + } + } + } for name, nodes := range seen { if len(nodes) > 1 { for _, node := range nodes { @@ -117,25 +734,25 @@ func checkUniqueInterfaceNames(g Grammar, accept core.ValidationAcceptor) { // TokenImpl.Validate checks terminal rule constraints: // - The regular expression should not match the empty string. -func (t *TokenImpl) Validate(_ context.Context, _ string, accept core.ValidationAcceptor) { +func (t *TokenDeclImpl) Validate(_ context.Context, _ string, accept core.ValidationAcceptor) { checkEmptyTerminalRule(t, accept) } -func checkEmptyTerminalRule(t Token, accept core.ValidationAcceptor) { - raw := t.Regexp() - if raw == "" { - return - } - // Strip surrounding slashes from the regex literal - pattern := raw - if len(pattern) >= 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' { - pattern = pattern[1 : len(pattern)-1] - } - re, err := regexp.Compile(pattern) - if err != nil { - return +func checkEmptyTerminalRule(t TokenDecl, accept core.ValidationAcceptor) { + var canBeEmpty bool + switch content := t.Content().(type) { + case KeywordTokenContent: + raw := KeywordValue(content.Keyword()) + canBeEmpty = raw == "" + case RegexpTokenContent: + pattern := RegexpValue(content.Regexp()) + re, err := regexp.Compile(pattern) + if err != nil { + return + } + canBeEmpty = re.MatchString("") } - if re.MatchString("") { + if canBeEmpty { accept(core.NewDiagnostic( core.SeverityError, "This terminal could match an empty string.", @@ -467,7 +1084,12 @@ func checkRuleCallReturnType(call RuleCall, ctx context.Context, accept core.Val } } -func checkRuleCallPosition(call RuleCall, _ context.Context, accept core.ValidationAcceptor) { +func checkRuleCallPosition(call RuleCall, ctx context.Context, accept core.ValidationAcceptor) { + rule := call.Rule().Ref(ctx) + if _, ok := rule.(ParserRule); !ok { + // Only parser rules can cause information loss, so we only check those + return + } // An unassigned rule call cannot be preceded by an action or assignment // This would lead to information loss, as the result of the rule call overrides the current AST node var node core.AstNode = call @@ -676,7 +1298,7 @@ func isAssignableTo(ctx context.Context, source Assignable, fieldType FieldType, return } switch rule := resolvedRule.(type) { - case Token: + case TokenDecl: if primitiveType, ok := fieldType.(PrimitiveType); !ok || primitiveType.Type() != "string" { accept(core.NewDiagnostic( core.SeverityError, @@ -773,7 +1395,10 @@ func doInterfaceIsAssignableTo(source Interface, target Interface, visited colle func (tg *TokenGroupImpl) Validate(_ context.Context, _ string, accept core.ValidationAcceptor) { checkRecursiveTokenGroup(tg, accept) - checkInvalidTokensInGroup(tg, accept) + checkTokenGroupContainsOnlyValidTokens(tg, accept) + for _, selector := range tg.KeywordSelectors() { + checkRegExpIsValid(selector, accept) + } } func checkRecursiveTokenGroup(tg TokenGroup, accept core.ValidationAcceptor) { @@ -788,6 +1413,24 @@ func checkRecursiveTokenGroup(tg TokenGroup, accept core.ValidationAcceptor) { } } +func checkTokenGroupContainsOnlyValidTokens(tg TokenGroup, accept core.ValidationAcceptor) { + for _, tokenRef := range tg.TokenRefs() { + token := tokenRef.Ref(context.Background()) + if token == nil { + continue + } + if description, special := hiddenOrCommentTokenDescription(token); special { + accept(core.NewDiagnostic( + core.SeverityError, + fmt.Sprintf("The token '%s' cannot be used in a token group because it is %s.", token.Name(), description), + tg, + core.WithReference(tokenRef), + core.WithCode(ValidateInvalidTokenInGroup), + )) + } + } +} + func appearsInTokenGroup(target TokenGroup, current TokenGroup, ctx context.Context, visited collections.Set[string]) bool { if !visited.Add(current.Name()) { return false @@ -805,8 +1448,8 @@ func appearsInTokenGroup(target TokenGroup, current TokenGroup, ctx context.Cont return false } -func hiddenOrCommentTokenDescription(token Token) (description string, ok bool) { - switch token.Type() { +func hiddenOrCommentTokenDescription(tokenDecl AbstractTokenRule) (description string, ok bool) { + switch tokenDecl.Modifier() { case "hidden": return "hidden", true case "comment": @@ -816,29 +1459,6 @@ func hiddenOrCommentTokenDescription(token Token) (description string, ok bool) } } -func checkInvalidTokensInGroup(tg TokenGroup, accept core.ValidationAcceptor) { - for _, ext := range tg.TokenRefs() { - abstractToken := ext.Ref(context.Background()) - if abstractToken == nil { - continue - } - if token, ok := abstractToken.(Token); ok { - // Hidden/comment tokens are not allowed in token groups. - // They are not meant to be consumed in parser rules, - // and do not appear in the token slice. - if description, special := hiddenOrCommentTokenDescription(token); special { - accept(core.NewDiagnostic( - core.SeverityError, - fmt.Sprintf("The token '%s' cannot be used in a token group because it is %s.", token.Name(), description), - tg, - core.WithReference(ext), - core.WithCode(ValidateInvalidTokenInGroup), - )) - } - } - } -} - func (cr *CrossRefImpl) Validate(ctx context.Context, _ string, accept core.ValidationAcceptor) { checkCrossRefHasTerminal(cr, ctx, accept) checkCrossRefToken(cr, ctx, accept) @@ -870,19 +1490,118 @@ func checkCrossRefToken(cr CrossRef, ctx context.Context, accept core.Validation if resolved == nil { return } - token, ok := resolved.(Token) + tokenDecl, ok := resolved.(TokenDecl) if !ok { return } // Hidden/comment tokens are not allowed in cross-references because they // are not stored in the token slice and cannot identify named elements. - if description, special := hiddenOrCommentTokenDescription(token); special { + if description, special := hiddenOrCommentTokenDescription(tokenDecl); special { accept(core.NewDiagnostic( core.SeverityError, - fmt.Sprintf("The token '%s' cannot be used in a cross-reference because it is %s.", token.Name(), description), + fmt.Sprintf("The token '%s' cannot be used in a cross-reference because it is %s.", tokenDecl.Name(), description), cr, core.WithReference(ruleCall.Rule()), core.WithCode(ValidateInvalidTokenInCrossRef), )) } } + +func checkTokenModeMembersAreUnique(tm TokenMode, accept core.ValidationAcceptor) { + seen := collections.NewSet[string]() + for _, member := range tm.Members() { + var name string + var textRange core.TextRange + var token *core.Token = nil + switch member := member.(type) { + case TokenDeclUsage: + continue // TokenDeclUsage is already checked in checkUniqueRuleNames + case TokenGroupUsage: + continue // TokenGroupUsage is already checked in checkUniqueRuleNames + case TokenUsage: + decl := member.TokenRef().Ref(context.Background()) + textRange = member.TextRange() + if decl == nil { + continue + } + name = decl.Name() + case KeywordUsage: + name = member.Keyword().Value() + token = member.Keyword().ValueToken() + default: + continue + } + if token != nil { + textRange = token.TextRange() + } + if !seen.Add(name) { + accept(core.NewDiagnostic( + core.SeverityError, + fmt.Sprintf("The token mode '%s' contains multiple members with the same name '%s'.", tm.Name(), name), + member, + core.WithTextRange(textRange), + core.WithCode(ValidateUniqueRuleNameInTokenMode), + )) + } + } +} + +func (m *KeywordSelectorImpl) Validate(_ context.Context, _ string, accept core.ValidationAcceptor) { + checkRegExpIsValid(m.SelectorToken(), accept) +} + +func (m *RegexpTokenContentImpl) Validate(_ context.Context, _ string, accept core.ValidationAcceptor) { + checkRegExpIsValid(m.RegexpToken(), accept) +} + +func checkRegExpIsValid(patternToken *core.Token, accept core.ValidationAcceptor) { + _, err := regexp.Compile(RegexpValue(patternToken.Image)) + if err != nil { + accept(core.NewDiagnostic( + core.SeverityError, + fmt.Sprintf("The keyword selector '%s' is not a valid regular expression: %s", patternToken.Image, err.Error()), + patternToken.Element, + core.WithToken(patternToken), + core.WithCode(ValidateInvalidRegExpLiteral), + )) + } +} + +func checkIfKeywordPureStandaloneOrTokenDecl(g Grammar, ctx context.Context, accept core.ValidationAcceptor) { + //value => Keyword => isTokenDeclaration + keywords := map[string]map[Keyword]bool{} + for node := range core.AllChildren(g) { + if kw, ok := node.(Keyword); ok { + value := kw.Value() + if _, exists := keywords[value]; !exists { + keywords[value] = map[Keyword]bool{} + } + _, ok := kw.Container().(KeywordTokenContent) + keywords[value][kw] = ok + } + } + + for _, kws := range keywords { + foundInDecl := false + foundStandalone := false + for _, isTokenDecl := range kws { + if isTokenDecl { + foundInDecl = true + } else { + foundStandalone = true + } + } + if !foundInDecl || !foundStandalone { + continue + } + for kw, _ := range kws { + accept(core.NewDiagnostic( + core.SeverityError, + fmt.Sprintf("The keyword '%s' is used both as a standalone keyword and as a token declaration. A unique keyword value must be either pure standalone or pure token declarations.", kw.Value()), + kw, + core.WithToken(kw.ValueToken()), + core.WithCode(ValidateKeywordPureStandaloneOrTokenDecl), + )) + } + } +} diff --git a/internal/grammar/validator_test.go b/internal/grammar/validator_test.go index 4eed9a21..e8b1460b 100644 --- a/internal/grammar/validator_test.go +++ b/internal/grammar/validator_test.go @@ -7,6 +7,7 @@ package grammar import ( "testing" + "github.com/stretchr/testify/assert" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/test" ) @@ -403,11 +404,15 @@ func TestTerminalMatchesEmptyString(t *testing.T) { grammar Test; token ID: /[a-zA-Z_][a-zA-Z0-9_]*/; token <|EMPTY|>: /a*/; + token <|EMPTY2|>: ""; hidden token WS: /[ \n\r\t]+/; `) diag := doc.ExpectDiagnostic("EMPTY") diag.WithSeverity(core.SeverityError) diag.WithCode(ValidateEmptyToken) + diag = doc.ExpectDiagnostic("EMPTY2") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateEmptyToken) } // --- Keywords --- @@ -505,7 +510,7 @@ func TestUnassignedRuleCallReturnTypeMismatch(t *testing.T) { diag.WithCode(ValidateRuleCallReturnType) } -func TestUnassignedRuleCallAfterAction(t *testing.T) { +func TestUnassignedRuleCallToParserRuleAfterAction(t *testing.T) { f := test.New(t, CreateServices()) // Bar extends Foo, so {Bar.Items+=current} is type-valid; the only error is the position check. doc := f.Parse(` @@ -519,7 +524,20 @@ func TestUnassignedRuleCallAfterAction(t *testing.T) { diag.WithCode(ValidateRuleCallPosition) } -func TestUnassignedRuleCallAfterAssignment(t *testing.T) { +func TestUnassignedRuleCallToNonParserRuleAfterAction(t *testing.T) { + f := test.New(t, CreateServices()) + // Bar extends Foo, so {Bar.Items+=current} is type-valid; the only error is the position check. + doc := f.Parse(` + grammar Test; + interface Foo { Items []Foo } + interface Bar extends Foo {} + Bar: ({Bar.Items+=current} <|Car|>); + token Car: "car"; + ` + commonTokens) + doc.AssertNoDiagnostic("Car") +} + +func TestUnassignedRuleCallToParserRuleAfterAssignment(t *testing.T) { f := test.New(t, CreateServices()) doc := f.Parse(` grammar Test; @@ -532,6 +550,17 @@ func TestUnassignedRuleCallAfterAssignment(t *testing.T) { diag.WithCode(ValidateRuleCallPosition) } +func TestUnassignedRuleCallToNonParserRuleAfterAssignment(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Name string } + Foo: Name=ID <|SubRule|>; + token SubRule: "subrule"; + ` + commonTokens) + doc.AssertNoDiagnostic("SubRule") +} + // --- Action type assignability --- func TestActionTypeNotAssignableToRuleReturn(t *testing.T) { @@ -782,3 +811,1040 @@ func TestCrossRefMissingTerminal(t *testing.T) { diag.WithSeverity(core.SeverityError) diag.WithCode(ValidateMissingCrossRefTerminal) } + +func TestDuplicateTokenModeNames(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting="hello"; + + token mode <|1:default|> { "hello" } + token mode <|2:default|> { } + `) + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityError) + diag1.WithCode(ValidateUniqueTokenModeName) + diag2 := doc.ExpectDiagnostic("2") + diag2.WithSeverity(core.SeverityError) + diag2.WithCode(ValidateUniqueTokenModeName) +} + +func TestDuplicateTokenNamesAcrossDifferentTokenScopes(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token <|1:ID|>: /[a-z_][a-z0-9_]*/ + hidden token WS: /[ \n\r\t]+/ + + token mode default { + token <|2:ID|>: /[A-Z_][A-Z0-9_]*/ + token group <|3:ID|> {} + } + `) + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityError) + diag1.WithCode(ValidateUniqueRuleName) + diag2 := doc.ExpectDiagnostic("2") + diag2.WithSeverity(core.SeverityError) + diag2.WithCode(ValidateUniqueRuleName) + diag3 := doc.ExpectDiagnostic("3") + diag3.WithSeverity(core.SeverityError) + diag3.WithCode(ValidateUniqueRuleName) +} + +func TestNonTokenModeRequiresDefaultTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[A-Z_][A-Z0-9_]*/ + + token mode <|1:FirstMode|> { + ID + } + `) + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityError) + diag1.WithCode(ValidateDefaultTokenModeRequired) +} + +// --- Token mode commands --- + +func TestPushCommandWithoutTargetMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> <|1:push|> + hidden WS + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateTokenCommandMode) + diag.WithMessageContaining("requires a target token mode") +} + +func TestModeCommandWithoutTargetMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> <|1:mode|> + hidden WS + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateTokenCommandMode) +} + +func TestPushCommandWithTargetModeIsValid(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(Other) + hidden WS + } + + token mode Other { + ID -> pop + } + `).AssertNoDiagnostics() +} + +func TestPushCommandTargetingDefaultModeIsValid(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(Other) + hidden WS + } + + token mode Other { + ID -> mode(default) + } + `).AssertNoDiagnostics() +} + +func TestPushCommandWithUnknownTargetMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(<|1:Missing|>) + hidden WS + } + `) + // The linker reports the dangling mode reference; the command itself has a + // target and must not additionally be flagged as incomplete. + doc.ExpectDiagnostic("1").WithSeverity(core.SeverityError) + for _, diag := range doc.Diagnostics() { + if diag.Code == ValidateTokenCommandMode { + t.Errorf("unexpected %s diagnostic: %s", ValidateTokenCommandMode, diag.Message) + } + } +} + +func TestPopCommandWithTargetMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(Other) + hidden WS + } + + token mode Other { + ID -> pop(<|1:default|>) + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateTokenCommandMode) + diag.WithMessageContaining("cannot take a target mode") +} + +func TestPopCommandWithTargetModeReference(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(Other) + hidden WS + } + + token mode Other { + ID -> pop(<|1:Other|>) + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateTokenCommandMode) +} + +// --- Token mode reachability and emptiness --- + +func TestUnreachableTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + + token mode <|1:Orphan|> { + ID + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityWarning) + diag.WithCode(ValidateUnreachableTokenMode) +} + +func TestUnreachableTokenModeNotReportedForPopTarget(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + + token mode <|1:Orphan|> { + ID -> pop + } + `) + // A 'pop' inside a mode does not make that mode reachable. + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityWarning) + diag.WithCode(ValidateUnreachableTokenMode) +} + +func TestDefaultTokenModeIsAlwaysReachable(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + `).AssertNoDiagnostics() +} + +func TestUnreachableTokenModeReachedFromNestedMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(Middle) + hidden WS + } + + token mode <|1:Middle|> { + ID -> push(Inner) + } + + token mode Inner { + ID -> pop + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityWarning) + diag.WithCode(ValidateNonDefaultTokenModeNoPop) +} + +func TestEmptyTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode <|1:default|> { } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityWarning) + diag.WithCode(ValidateEmptyTokenMode) +} + +func TestEmptyNamedTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(Other) + hidden WS + } + + token mode <|1:Other|> { } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityWarning) + diag.WithCode(ValidateEmptyTokenMode) + diag.WithMessageContaining("'Other'") +} + +// --- Token mode coverage of parser tokens --- + +func TestKeywordNotInAnyTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID <|1:"world"|>; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateKeywordNotInTokenMode) +} + +func TestKeywordListedInTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID "world"; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + "world" + hidden WS + } + `).AssertNoDiagnostics() +} + +func TestKeywordCoveredByKeywordSelector(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID "world"; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + keywords /^[a-z]+$/ + ID + hidden WS + } + `).AssertNoDiagnostics() +} + +func TestKeywordNotMatchedByKeywordSelector(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID <|1:"!"|>; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + keywords /^[a-z]+$/ + ID + hidden WS + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateKeywordNotInTokenMode) +} + +func TestKeywordCoveredByKeywordBackedToken(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID WORLD; + + token WORLD: "world" + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + WORLD + hidden WS + } + `).AssertNoErrors() +} + +func TestKeywordCoveredByTokenGroupInMode(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID "world"; + + token group Greetings { + "world" + } + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + Greetings + hidden WS + } + `).AssertNoErrors() +} + +func TestKeywordCoveredByNestedTokenGroupInMode(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID "world"; + + token group Inner { + "world" + } + token group Outer { + Inner + } + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + Outer + hidden WS + } + `).AssertNoErrors() +} + +func TestKeywordCoveredByModeLocalTokenDeclaration(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID WORLD; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + token WORLD: "world" + hidden WS + } + `).AssertNoErrors() +} + +func TestKeywordCoveredByNonDefaultTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID "world"; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID -> push(Other) + hidden WS + } + + token mode Other { + "world" -> pop + } + `).AssertNoDiagnostics() +} + +func TestKeywordInTokenDeclarationNotReportedAsUncovered(t *testing.T) { + f := test.New(t, CreateServices()) + // GREETING is never used by a parser rule, so its keyword does not have to + // be reachable through a token mode. + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token GREETING: "hello" + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + `).AssertNoErrors() +} + +func TestKeywordUncoveredReportedOnlyOnce(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID "world" "world" "world"; + + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + `) + count := 0 + for _, diag := range doc.Diagnostics() { + if diag.Code == ValidateKeywordNotInTokenMode { + count++ + } + } + if count != 1 { + t.Errorf("expected 1 %s diagnostic, got %d", ValidateKeywordNotInTokenMode, count) + } +} + +func TestKeywordCoverageNotCheckedWithoutTokenModes(t *testing.T) { + f := test.New(t, CreateServices()) + // Without token modes the generator registers every keyword automatically. + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID "world"; + ` + commonTokens).AssertNoDiagnostics() +} + +func TestTokenNotInAnyTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string Name string } + Foo: Greeting=NAME Name=ID; + + token <|1:NAME|>: /[A-Z]+/ + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityWarning) + diag.WithCode(ValidateTokenNotInTokenMode) + diag.WithMessageContaining("'NAME'") +} + +func TestTokenGroupNotInAnyTokenModeButMembersWereConsumed(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=Greetings; + + token group <|1:Greetings|> { + ID + } + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + `) + // The group's members are registered, the group has its own token id. + // But it is reachable because one of its members is used in a parser rule. + doc.AssertNoDiagnostic("1") +} + +func TestTokenGroupNotInAnyTokenModeAndMembersWereNotConsumed(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=Greetings; + + token group <|1:Greetings|> { + "any" + "thing" + } + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + `) + // The group's members are registered, the group has its own token id. + // It is also not reachable because none of its members is used in a parser rule. + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityWarning) + diag.WithCode(ValidateTokenNotInTokenMode) +} + +func TestKeywordBackedTokenCoveredByKeywordSelector(t *testing.T) { + f := test.New(t, CreateServices()) + // WORLD shares the token id of the keyword "world", so a selector that + // covers the keyword covers the token too. + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=WORLD; + + token WORLD: "world" + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + keywords /^[a-z]+$/ + ID + hidden WS + } + `).AssertNoErrors() +} + +func TestTokenInCrossRefNotInAnyTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Name string } + interface Bar { Ref *Foo } + Foo: Name=ID; + Bar: Ref=[Foo:NAME]; + + token <|1:NAME|>: /[A-Z]+/ + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityWarning) + diag.WithCode(ValidateTokenNotInTokenMode) +} + +func TestTokenInCompositeRuleNotInAnyTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Name composite } + Foo: Name=FQN; + composite FQN: ID (DOT ID)*; + + token <|1:DOT|>: /\./ + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + ID + hidden WS + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityWarning) + diag.WithCode(ValidateTokenNotInTokenMode) +} + +func TestTokenOnlyUsedInsideTokenGroupNotReported(t *testing.T) { + f := test.New(t, CreateServices()) + // NAME is referenced from a token group, not from a parser rule, so it does + // not need its own entry in a token mode. + f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=Greetings; + + token group Greetings { + ID + NAME + } + token NAME: /[A-Z]+/ + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + Greetings + hidden WS + } + `).AssertNoDiagnostics() +} + +func TestRecursiveTokenGroupCoverageTerminates(t *testing.T) { + f := test.New(t, CreateServices()) + // The recursion is reported elsewhere; coverage collection must not hang. + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=Outer; + + token group Outer { + Inner + } + token group Inner { + Outer + ID + } + token ID: /[a-z]+/ + hidden token WS: /\s+/ + + token mode default { + Outer + hidden WS + } + `) + for _, diag := range doc.Diagnostics() { + if diag.Code == ValidateTokenNotInTokenMode { + t.Errorf("unexpected %s diagnostic: %s", ValidateTokenNotInTokenMode, diag.Message) + } + } +} + +func TestUnreachableTokenModeCycle(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + + token mode <|0:default|> { + ID + } + + token mode <|1:LEFT|> { + ID -> mode(RIGHT) + } + token mode <|2:RIGHT|> { + ID -> mode(LEFT) + } + `) + doc.AssertNoDiagnostic("0") + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityWarning) + diag1.WithCode(ValidateUnreachableTokenMode) + diag2 := doc.ExpectDiagnostic("2") + diag2.WithSeverity(core.SeverityWarning) + diag2.WithCode(ValidateUnreachableTokenMode) +} + +func TestNonDefaultTokenModeWithNoExitCommand(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + + token mode <|0:default|> { + ID -> push(LEFT) + } + + token mode <|1:LEFT|> { + ID + "123" + } + `) + doc.AssertNoDiagnostic("0") + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityWarning) + diag1.WithCode(ValidateNonDefaultTokenModeNoPop) +} + +func TestTokenNotUsedInTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token <|1:ID|>: /[a-z]+/ + + token mode default { "hello" } + `) + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityWarning) + diag1.WithCode(ValidateTokenNotInTokenMode) +} + +func TestMembersUniqueInTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=ID; + + token ID: /[a-z]+/ + + token mode default { + ID + <|1:ID|> + + "hello" + <|2:"hello"|> + + token X: /lol/ + token <|3:X|>: /lol2/ + + token group G { + ID + } + token group <|4:G|> { + ID + } + } + `) + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityError) + diag1.WithCode(ValidateUniqueRuleNameInTokenMode) + diag2 := doc.ExpectDiagnostic("2") + diag2.WithSeverity(core.SeverityError) + diag2.WithCode(ValidateUniqueRuleNameInTokenMode) + diag3 := doc.ExpectDiagnostic("3") + diag3.WithSeverity(core.SeverityError) + diag3.WithCode(ValidateUniqueRuleName) + diag4 := doc.ExpectDiagnostic("4") + diag4.WithSeverity(core.SeverityError) + diag4.WithCode(ValidateUniqueRuleName) +} + +func TestNotCoveredByParserRuleUsingTokenModes(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting="x"; + + token NUM: /[0-9]+/ + + token mode default { + token <|1:ID|>: /[a-z]+/ + <|2:NUM|> + token group <|3:Cardinality|> { + "?" + "*" + "+" + } + <|4:"unused"|> + } + `) + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityWarning) + diag1.WithCode(ValidateTerminalNotCoveredByParserRule) + diag2 := doc.ExpectDiagnostic("2") + diag2.WithSeverity(core.SeverityWarning) + diag2.WithCode(ValidateTerminalNotCoveredByParserRule) + diag3 := doc.ExpectDiagnostic("3") + diag3.WithSeverity(core.SeverityWarning) + diag3.WithCode(ValidateTokenGroupNotCoveredByParserRule) + diag4 := doc.ExpectDiagnostic("4") + diag4.WithSeverity(core.SeverityWarning) + diag4.WithCode(ValidateTerminalNotCoveredByParserRule) +} + +func TestNotCoveredByParserRuleUsingImplicitTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting="x"; + + token <|1:ID|>: /[a-z]+/ + token group <|2:Cardinality|> { + "?" + "*" + "+" + } + `) + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityWarning) + diag1.WithCode(ValidateTerminalNotCoveredByParserRule) + diag2 := doc.ExpectDiagnostic("2") + diag2.WithSeverity(core.SeverityWarning) + diag2.WithCode(ValidateTokenGroupNotCoveredByParserRule) +} + +func TestInvisibleTokenDontNeedToBeCoveredByParserRule(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting="x"; + + hidden token WS: /\s+/ + comment token group Comment { + "//comment" + "/*comment*/" + } + `) + doc.AssertNoDiagnostics() +} + +func TestInvalidKeywordSelectorInTokenGroup(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=XX X; + + token XX: "x" + + token group X { + //BROKEN REGEXP!!! + keywords <|1:/(/|> + } + `) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateInvalidRegExpLiteral) +} + +func TestInvalidKeywordSelectorInTokenMode(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting="x"; + + token mode default { + //BROKEN REGEXP!!! + keywords <|1:/(/|> + "x" + } + `) + assert.Len(t, doc.Document.Diagnostics, 1) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateInvalidRegExpLiteral) +} + +func TestInvalidRegExpInTokenContent(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Greeting string } + Foo: Greeting=Y; + token Y: <|1:/(/|> + `) + assert.Len(t, doc.Document.Diagnostics, 1) + diag := doc.ExpectDiagnostic("1") + diag.WithSeverity(core.SeverityError) + diag.WithCode(ValidateInvalidRegExpLiteral) +} + +func TestSameKeywordsAsStandaloneAndTokenDeclaration(t *testing.T) { + f := test.New(t, CreateServices()) + doc := f.Parse(` + grammar Test; + interface Foo { Bar string } + Foo: Bar=Y <|1:"y"|>; + token Y: <|2:"y"|> + `) + assert.Len(t, doc.Document.Diagnostics, 2) + diag1 := doc.ExpectDiagnostic("1") + diag1.WithSeverity(core.SeverityError) + diag1.WithCode(ValidateKeywordPureStandaloneOrTokenDecl) + diag2 := doc.ExpectDiagnostic("2") + diag2.WithSeverity(core.SeverityError) + diag2.WithCode(ValidateKeywordPureStandaloneOrTokenDecl) +} diff --git a/internal/languages/completion/atn.md b/internal/languages/completion/atn.md new file mode 100644 index 00000000..180fcb2e --- /dev/null +++ b/internal/languages/completion/atn.md @@ -0,0 +1,564 @@ +# Runtime ATN for completion + +## Root + +```mermaid +flowchart TD + q0(["Root__Start (0)
RuleStart"]) + q1(["Root__Stop (1)
RuleStop"]) + q46["Root__Basic_0 (46)
Basic
"] + q47["Root__Basic_1 (47)
Basic
"] + q48["Root__Basic_2 (48)
Basic
"] + q49["Root__Basic_3 (49)
Basic
"] + q50["Root__Basic_4 (50)
Basic
"] + q51["Root__Basic_5 (51)
Basic
"] + q52["Root__Basic_6 (52)
Basic
"] + q53["Root__Basic_7 (53)
Basic
"] + q54["Root__Basic_8 (54)
Basic
"] + q55["Root__Basic_9 (55)
Basic
"] + q56["Root__Basic_10 (56)
Basic
"] + q57["Root__Basic_11 (57)
Basic
"] + q58["Root__Basic_12 (58)
Basic
"] + q59["Root__Basic_13 (59)
Basic
"] + q60["Root__Basic_14 (60)
Basic
"] + q61["Root__Basic_15 (61)
Basic
"] + q62["Root__Basic_16 (62)
Basic
"] + q63["Root__Basic_17 (63)
Basic
"] + q64["Root__Basic_18 (64)
Basic
"] + q65["Root__Basic_19 (65)
Basic
"] + q66["Root__Basic_20 (66)
Basic
"] + q67["Root__Basic_21 (67)
Basic
"] + q68["Root__Basic_22 (68)
Basic
"] + q69["Root__Basic_23 (69)
Basic
"] + q70["Root__Basic_24 (70)
Basic
"] + q71["Root__Basic_25 (71)
Basic
"] + q72["Root__Basic_26 (72)
Basic
"] + q73["Root__Basic_27 (73)
Basic
"] + q74["Root__Basic_28 (74)
Basic
"] + q75["Root__Basic_29 (75)
Basic
"] + q76["Root__Basic_30 (76)
Basic
"] + q77["Root__Basic_31 (77)
Basic
"] + q78{"Root__Basic_32 (78)
Basic

dec=0"} + q79["Root__BlockEnd (79)
BlockEnd
"] + q80{"Root__LoopEntry (80)
LoopEntry

dec=1"} + q81["Root__LoopEnd (81)
LoopEnd
"] + q82["Root__LoopBack (82)
LoopBack
"] + + q0 --> q80 + q46 -.->|"[Declare]"| q47 + q47 --> q79 + q48 -.->|"[A]"| q49 + q49 --> q79 + q50 -.->|"[B]"| q51 + q51 --> q79 + q52 -.->|"[C]"| q53 + q53 --> q79 + q54 -.->|"[D]"| q55 + q55 --> q79 + q56 -.->|"[E]"| q57 + q57 --> q79 + q58 -.->|"[F]"| q59 + q59 --> q79 + q60 -.->|"[G]"| q61 + q61 --> q79 + q62 -.->|"[H]"| q63 + q63 --> q79 + q64 -.->|"[I]"| q65 + q65 --> q79 + q66 -.->|"[J]"| q67 + q67 --> q79 + q68 -.->|"[K]"| q69 + q69 --> q79 + q70 -.->|"[L]"| q71 + q71 --> q79 + q72 -.->|"[M]"| q73 + q73 --> q79 + q74 -.->|"[N]"| q75 + q75 --> q79 + q76 -.->|"[O]"| q77 + q77 --> q79 + q78 --> q46 + q78 --> q48 + q78 --> q50 + q78 --> q52 + q78 --> q54 + q78 --> q56 + q78 --> q58 + q78 --> q60 + q78 --> q62 + q78 --> q64 + q78 --> q66 + q78 --> q68 + q78 --> q70 + q78 --> q72 + q78 --> q74 + q78 --> q76 + q79 --> q82 + q80 --> q78 + q80 --> q81 + q81 --> q1 + q82 --> q80 +``` + +## Declare + +```mermaid +flowchart TD + q2(["Declare__Start (2)
RuleStart"]) + q3(["Declare__Stop (3)
RuleStop"]) + q83["Declare_DECLARE (83)
Basic
"] + q84["Declare__Basic_0 (84)
Basic
"] + q85["Declare_LBRACE (85)
Basic
"] + q86["Declare__Basic_1 (86)
Basic
"] + q87["Declare__Basic_2 (87)
Basic
"] + q88{"Declare__LoopEntry (88)
LoopEntry

dec=2"} + q89["Declare__LoopEnd (89)
LoopEnd
"] + q90["Declare__LoopBack (90)
LoopBack
"] + q91["Declare_RBRACE (91)
Basic
"] + q92["Declare__Basic_3 (92)
Basic
"] + q93{"Declare__Basic_4 (93)
Basic

dec=3"} + + q2 --> q83 + q83 -->|"tok(Token_DECLARE)"| q84 + q84 -.->|"[FQN]"| q93 + q85 -->|"tok(Token_LBRACE)"| q88 + q86 -.->|"[Declare]"| q87 + q87 --> q90 + q88 --> q86 + q88 --> q89 + q89 --> q91 + q90 --> q88 + q91 -->|"tok(Token_RBRACE)"| q92 + q92 --> q3 + q93 --> q85 + q93 --> q92 +``` + +## A + +```mermaid +flowchart TD + q4(["A__Start (4)
RuleStart"]) + q5(["A__Stop (5)
RuleStop"]) + q94["A_a (94)
Basic
"] + q95["A_FIRST (95)
Basic
"] + q96["A__Basic (96)
Basic
"] + + q4 --> q94 + q94 -->|"tok(Keyword_a)"| q95 + q95 -->|"tok(Token_FIRST)"| q96 + q96 --> q5 +``` + +## B + +```mermaid +flowchart TD + q6(["B__Start (6)
RuleStart"]) + q7(["B__Stop (7)
RuleStop"]) + q97["B_b (97)
Basic
"] + q98["B_FIRST (98)
Basic
"] + q99["B__Basic_0 (99)
Basic
"] + q100["B_SECOND (100)
Basic
"] + q101["B__Basic_1 (101)
Basic
"] + q102{"B__Basic_2 (102)
Basic

dec=4"} + q103["B__BlockEnd (103)
BlockEnd
"] + + q6 --> q97 + q97 -->|"tok(Keyword_b)"| q102 + q98 -->|"tok(Token_FIRST)"| q99 + q99 --> q103 + q100 -->|"tok(Token_SECOND)"| q101 + q101 --> q103 + q102 --> q98 + q102 --> q100 + q103 --> q7 +``` + +## C + +```mermaid +flowchart TD + q8(["C__Start (8)
RuleStart"]) + q9(["C__Stop (9)
RuleStop"]) + q104["C_c (104)
Basic
"] + q105["C_COMMON_0 (105)
Basic
"] + q106["C_FIRST (106)
Basic
"] + q107["C__Basic_0 (107)
Basic
"] + q108["C_COMMON_1 (108)
Basic
"] + q109["C_SECOND (109)
Basic
"] + q110["C__Basic_1 (110)
Basic
"] + q111{"C__Basic_2 (111)
Basic

dec=5"} + q112["C__BlockEnd (112)
BlockEnd
"] + + q8 --> q104 + q104 -->|"tok(Keyword_c)"| q111 + q105 -->|"tok(Token_COMMON)"| q106 + q106 -->|"tok(Token_FIRST)"| q107 + q107 --> q112 + q108 -->|"tok(Token_COMMON)"| q109 + q109 -->|"tok(Token_SECOND)"| q110 + q110 --> q112 + q111 --> q105 + q111 --> q108 + q112 --> q9 +``` + +## D + +```mermaid +flowchart TD + q10(["D__Start (10)
RuleStart"]) + q11(["D__Stop (11)
RuleStop"]) + q113["D_d (113)
Basic
"] + q114["D__Basic_0 (114)
Basic
"] + q115["D__Basic_1 (115)
Basic
"] + q116["D__Basic_2 (116)
Basic
"] + q117["D__Basic_3 (117)
Basic
"] + q118{"D__Basic_4 (118)
Basic

dec=6"} + q119["D__BlockEnd (119)
BlockEnd
"] + + q10 --> q113 + q113 -->|"tok(Keyword_d)"| q118 + q114 -.->|"[DLong]"| q115 + q115 --> q119 + q116 -.->|"[DShort]"| q117 + q117 --> q119 + q118 --> q114 + q118 --> q116 + q119 --> q11 +``` + +## E + +```mermaid +flowchart TD + q12(["E__Start (12)
RuleStart"]) + q13(["E__Stop (13)
RuleStop"]) + q120["E_e (120)
Basic
"] + q121["E__Basic_0 (121)
Basic
"] + q122["E__Basic_1 (122)
Basic
"] + + q12 --> q120 + q120 -->|"tok(Keyword_e)"| q121 + q121 -.->|"[FQN]"| q122 + q122 --> q13 +``` + +## DLong + +```mermaid +flowchart TD + q14(["DLong__Start (14)
RuleStart"]) + q15(["DLong__Stop (15)
RuleStop"]) + q123["DLong_COMMON (123)
Basic
"] + q124["DLong_THEN (124)
Basic
"] + q125["DLong_LONG (125)
Basic
"] + q126["DLong__Basic (126)
Basic
"] + + q14 --> q123 + q123 -->|"tok(Token_COMMON)"| q124 + q124 -->|"tok(Token_THEN)"| q125 + q125 -->|"tok(Token_LONG)"| q126 + q126 --> q15 +``` + +## DShort + +```mermaid +flowchart TD + q16(["DShort__Start (16)
RuleStart"]) + q17(["DShort__Stop (17)
RuleStop"]) + q127["DShort_COMMON (127)
Basic
"] + q128["DShort__Basic (128)
Basic
"] + + q16 --> q127 + q127 -->|"tok(Token_COMMON)"| q128 + q128 --> q17 +``` + +## F + +```mermaid +flowchart TD + q18(["F__Start (18)
RuleStart"]) + q19(["F__Stop (19)
RuleStop"]) + q129["F_f (129)
Basic
"] + q130["F__Basic_0 (130)
Basic
"] + q131["F__Basic_1 (131)
Basic
"] + q132{"F__LoopBack (132)
LoopBack

dec=7"} + q133["F__LoopEnd (133)
LoopEnd
"] + + q18 --> q129 + q129 -->|"tok(Keyword_f)"| q130 + q130 -.->|"[FItem]"| q131 + q131 --> q132 + q132 --> q130 + q132 --> q133 + q133 --> q19 +``` + +## FItem + +```mermaid +flowchart TD + q20(["FItem__Start (20)
RuleStart"]) + q21(["FItem__Stop (21)
RuleStop"]) + q134["FItem__Basic_0 (134)
Basic
"] + q135["FItem__Basic_1 (135)
Basic
"] + + q20 --> q134 + q134 -.->|"[FQN]"| q135 + q135 --> q21 +``` + +## G + +```mermaid +flowchart TD + q22(["G__Start (22)
RuleStart"]) + q23(["G__Stop (23)
RuleStop"]) + q136["G_g (136)
Basic
"] + q137["G_Ref_ID (137)
Basic
"] + q138["G__Basic (138)
Basic
"] + + q22 --> q136 + q136 -->|"tok(Keyword_g)"| q137 + q137 -->|"tok(Token_ID)"| q138 + q138 --> q23 +``` + +## H + +```mermaid +flowchart TD + q24(["H__Start (24)
RuleStart"]) + q25(["H__Stop (25)
RuleStop"]) + q139["H_h (139)
Basic
"] + q140["H__Basic_0 (140)
Basic
"] + q141["H__Basic_1 (141)
Basic
"] + + q24 --> q139 + q139 -->|"tok(Keyword_h)"| q140 + q140 -.->|"[MemberCall]"| q141 + q141 --> q25 +``` + +## I + +```mermaid +flowchart TD + q26(["I__Start (26)
RuleStart"]) + q27(["I__Stop (27)
RuleStop"]) + q142["I_i (142)
Basic
"] + q143["I__Basic_0 (143)
Basic
"] + q144["I__Basic_1 (144)
Basic
"] + + q26 --> q142 + q142 -->|"tok(Keyword_i)"| q143 + q143 -.->|"[MemberCallNoDot]"| q144 + q144 --> q27 +``` + +## MemberCall + +```mermaid +flowchart TD + q28(["MemberCall__Start (28)
RuleStart"]) + q29(["MemberCall__Stop (29)
RuleStop"]) + q145["MemberCall_Ref_ID_0 (145)
Basic
"] + q146["MemberCall_DOT (146)
Basic
"] + q147["MemberCall_Ref_ID_1 (147)
Basic
"] + q148["MemberCall__Basic (148)
Basic
"] + q149{"MemberCall__LoopEntry (149)
LoopEntry

dec=8"} + q150["MemberCall__LoopEnd (150)
LoopEnd
"] + q151["MemberCall__LoopBack (151)
LoopBack
"] + + q28 --> q145 + q145 -->|"tok(Token_ID)"| q149 + q146 -->|"tok(Token_DOT)"| q147 + q147 -->|"tok(Token_ID)"| q148 + q148 --> q151 + q149 --> q146 + q149 --> q150 + q150 --> q29 + q151 --> q149 +``` + +## MemberCallNoDot + +```mermaid +flowchart TD + q30(["MemberCallNoDot__Start (30)
RuleStart"]) + q31(["MemberCallNoDot__Stop (31)
RuleStop"]) + q152["MemberCallNoDot_Ref_ID_0 (152)
Basic
"] + q153["MemberCallNoDot_Ref_ID_1 (153)
Basic
"] + q154["MemberCallNoDot__Basic (154)
Basic
"] + q155{"MemberCallNoDot__LoopEntry (155)
LoopEntry

dec=9"} + q156["MemberCallNoDot__LoopEnd (156)
LoopEnd
"] + q157["MemberCallNoDot__LoopBack (157)
LoopBack
"] + + q30 --> q152 + q152 -->|"tok(Token_ID)"| q155 + q153 -->|"tok(Token_ID)"| q154 + q154 --> q157 + q155 --> q153 + q155 --> q156 + q156 --> q31 + q157 --> q155 +``` + +## J + +```mermaid +flowchart TD + q32(["J__Start (32)
RuleStart"]) + q33(["J__Stop (33)
RuleStop"]) + q158["J_j (158)
Basic
"] + q159["J_Ref_ID (159)
Basic
"] + q160["J__Basic_0 (160)
Basic
"] + q161["J_SELF (161)
Basic
"] + q162["J__Basic_1 (162)
Basic
"] + q163{"J__Basic_2 (163)
Basic

dec=10"} + q164["J__BlockEnd (164)
BlockEnd
"] + + q32 --> q158 + q158 -->|"tok(Keyword_j)"| q163 + q159 -->|"tok(Token_ID)"| q160 + q160 --> q164 + q161 -->|"tok(Token_SELF)"| q162 + q162 --> q164 + q163 --> q159 + q163 --> q161 + q164 --> q33 +``` + +## K + +```mermaid +flowchart TD + q34(["K__Start (34)
RuleStart"]) + q35(["K__Stop (35)
RuleStop"]) + q165["K_k (165)
Basic
"] + q166["K_Ref1_ID (166)
Basic
"] + q167["K_x (167)
Basic
"] + q168["K__Basic_0 (168)
Basic
"] + q169["K_Ref2_ID (169)
Basic
"] + q170["K_y (170)
Basic
"] + q171["K__Basic_1 (171)
Basic
"] + q172{"K__Basic_2 (172)
Basic

dec=11"} + q173["K__BlockEnd (173)
BlockEnd
"] + + q34 --> q165 + q165 -->|"tok(Keyword_k)"| q172 + q166 -->|"tok(Token_ID)"| q167 + q167 -->|"tok(Keyword_x)"| q168 + q168 --> q173 + q169 -->|"tok(Token_ID)"| q170 + q170 -->|"tok(Keyword_y)"| q171 + q171 --> q173 + q172 --> q166 + q172 --> q169 + q173 --> q35 +``` + +## L + +```mermaid +flowchart TD + q36(["L__Start (36)
RuleStart"]) + q37(["L__Stop (37)
RuleStop"]) + q174["L_l (174)
Basic
"] + q175["L_OPTIONAL (175)
Basic
"] + q176["L_AND (176)
Basic
"] + q177["L__Basic_0 (177)
Basic
"] + q178{"L__Basic_1 (178)
Basic

dec=12"} + q179["L_THEN (179)
Basic
"] + q180["L_END (180)
Basic
"] + q181["L__Basic_2 (181)
Basic
"] + + q36 --> q174 + q174 -->|"tok(Keyword_l)"| q178 + q175 -->|"tok(Token_OPTIONAL)"| q176 + q176 -->|"tok(Token_AND)"| q177 + q177 --> q179 + q178 --> q175 + q178 --> q177 + q179 -->|"tok(Token_THEN)"| q180 + q180 -->|"tok(Token_END)"| q181 + q181 --> q37 +``` + +## M + +```mermaid +flowchart TD + q38(["M__Start (38)
RuleStart"]) + q39(["M__Stop (39)
RuleStop"]) + q182["M_m (182)
Basic
"] + q183["M__Basic_0 (183)
Basic
"] + q184["M__Basic_1 (184)
Basic
"] + + q38 --> q182 + q182 -->|"tok(Keyword_m)"| q183 + q183 -->|"tok(TokenGroup_SomeTokenGroup)"| q184 + q184 --> q39 +``` + +## N + +```mermaid +flowchart TD + q40(["N__Start (40)
RuleStart"]) + q41(["N__Stop (41)
RuleStop"]) + q185["N_n (185)
Basic
"] + q186["N__Basic_0 (186)
Basic
"] + q187["N__Basic_1 (187)
Basic
"] + + q40 --> q185 + q185 -->|"tok(Keyword_n)"| q186 + q186 -->|"tok(TokenGroup_SomeTokenGroup)"| q187 + q187 --> q41 +``` + +## O + +```mermaid +flowchart TD + q42(["O__Start (42)
RuleStart"]) + q43(["O__Stop (43)
RuleStop"]) + q188["O_o (188)
Basic
"] + q189["O_Ref_ID (189)
Basic
"] + q190["O__Basic (190)
Basic
"] + + q42 --> q188 + q188 -->|"tok(Keyword_o)"| q189 + q189 -->|"tok(Token_ID)"| q190 + q190 --> q43 +``` + +## FQN + +```mermaid +flowchart TD + q44(["FQN__Start (44)
RuleStart"]) + q45(["FQN__Stop (45)
RuleStop"]) + q191["FQN_ID_0 (191)
Basic
"] + q192["FQN_DOT (192)
Basic
"] + q193["FQN_ID_1 (193)
Basic
"] + q194["FQN__Basic (194)
Basic
"] + q195{"FQN__LoopEntry (195)
LoopEntry

dec=13"} + q196["FQN__LoopEnd (196)
LoopEnd
"] + q197["FQN__LoopBack (197)
LoopBack
"] + + q44 --> q191 + q191 -->|"tok(Token_ID)"| q195 + q192 -->|"tok(Token_DOT)"| q193 + q193 -->|"tok(Token_ID)"| q194 + q194 --> q197 + q195 --> q192 + q195 --> q196 + q196 --> q45 + q197 --> q195 +``` + diff --git a/internal/languages/completion/atn_gen.go b/internal/languages/completion/atn_gen.go index 80ba845a..e209629e 100644 --- a/internal/languages/completion/atn_gen.go +++ b/internal/languages/completion/atn_gen.go @@ -91,33 +91,33 @@ const ( Root__LoopEntry Root__LoopEnd Root__LoopBack - Declare_declare + Declare_DECLARE Declare__Basic_0 - Declare_LeftBrace + Declare_LBRACE Declare__Basic_1 Declare__Basic_2 Declare__LoopEntry Declare__LoopEnd Declare__LoopBack - Declare_RightBrace + Declare_RBRACE Declare__Basic_3 Declare__Basic_4 A_a - A_first + A_FIRST A__Basic B_b - B_first + B_FIRST B__Basic_0 - B_second + B_SECOND B__Basic_1 B__Basic_2 B__BlockEnd C_c - C_common_0 - C_first + C_COMMON_0 + C_FIRST C__Basic_0 - C_common_1 - C_second + C_COMMON_1 + C_SECOND C__Basic_1 C__Basic_2 C__BlockEnd @@ -131,11 +131,11 @@ const ( E_e E__Basic_0 E__Basic_1 - DLong_common - DLong_then - DLong_long + DLong_COMMON + DLong_THEN + DLong_LONG DLong__Basic - DShort_common + DShort_COMMON DShort__Basic F_f F__Basic_0 @@ -154,7 +154,7 @@ const ( I__Basic_0 I__Basic_1 MemberCall_Ref_ID_0 - MemberCall_Dot + MemberCall_DOT MemberCall_Ref_ID_1 MemberCall__Basic MemberCall__LoopEntry @@ -169,7 +169,7 @@ const ( J_j J_Ref_ID J__Basic_0 - J_self + J_SELF J__Basic_1 J__Basic_2 J__BlockEnd @@ -183,12 +183,12 @@ const ( K__Basic_2 K__BlockEnd L_l - L_optional - L_and + L_OPTIONAL + L_AND L__Basic_0 L__Basic_1 - L_then - L_end + L_THEN + L_END L__Basic_2 M_m M__Basic_0 @@ -200,7 +200,7 @@ const ( O_Ref_ID O__Basic FQN_ID_0 - FQN_Dot + FQN_DOT FQN_ID_1 FQN__Basic FQN__LoopEntry @@ -302,33 +302,33 @@ func BuildATN() *parser.RuntimeATN { states[Root__LoopEntry] = parser.NewATNState(Root__LoopEntry, parser.ATNLoopEntry, true).SetDecision(1) states[Root__LoopEnd] = parser.NewATNState(Root__LoopEnd, parser.ATNLoopEnd, true) states[Root__LoopBack] = parser.NewATNState(Root__LoopBack, parser.ATNLoopBack, true) - states[Declare_declare] = parser.NewATNState(Declare_declare, parser.ATNBasic, false) + states[Declare_DECLARE] = parser.NewATNState(Declare_DECLARE, parser.ATNBasic, false) states[Declare__Basic_0] = parser.NewATNState(Declare__Basic_0, parser.ATNBasic, true) - states[Declare_LeftBrace] = parser.NewATNState(Declare_LeftBrace, parser.ATNBasic, false) + states[Declare_LBRACE] = parser.NewATNState(Declare_LBRACE, parser.ATNBasic, false) states[Declare__Basic_1] = parser.NewATNState(Declare__Basic_1, parser.ATNBasic, true) states[Declare__Basic_2] = parser.NewATNState(Declare__Basic_2, parser.ATNBasic, true) states[Declare__LoopEntry] = parser.NewATNState(Declare__LoopEntry, parser.ATNLoopEntry, true).SetDecision(2) states[Declare__LoopEnd] = parser.NewATNState(Declare__LoopEnd, parser.ATNLoopEnd, true) states[Declare__LoopBack] = parser.NewATNState(Declare__LoopBack, parser.ATNLoopBack, true) - states[Declare_RightBrace] = parser.NewATNState(Declare_RightBrace, parser.ATNBasic, false) + states[Declare_RBRACE] = parser.NewATNState(Declare_RBRACE, parser.ATNBasic, false) states[Declare__Basic_3] = parser.NewATNState(Declare__Basic_3, parser.ATNBasic, true) states[Declare__Basic_4] = parser.NewATNState(Declare__Basic_4, parser.ATNBasic, true).SetDecision(3) states[A_a] = parser.NewATNState(A_a, parser.ATNBasic, false) - states[A_first] = parser.NewATNState(A_first, parser.ATNBasic, false) + states[A_FIRST] = parser.NewATNState(A_FIRST, parser.ATNBasic, false) states[A__Basic] = parser.NewATNState(A__Basic, parser.ATNBasic, true) states[B_b] = parser.NewATNState(B_b, parser.ATNBasic, false) - states[B_first] = parser.NewATNState(B_first, parser.ATNBasic, false) + states[B_FIRST] = parser.NewATNState(B_FIRST, parser.ATNBasic, false) states[B__Basic_0] = parser.NewATNState(B__Basic_0, parser.ATNBasic, true) - states[B_second] = parser.NewATNState(B_second, parser.ATNBasic, false) + states[B_SECOND] = parser.NewATNState(B_SECOND, parser.ATNBasic, false) states[B__Basic_1] = parser.NewATNState(B__Basic_1, parser.ATNBasic, true) states[B__Basic_2] = parser.NewATNState(B__Basic_2, parser.ATNBasic, true).SetDecision(4) states[B__BlockEnd] = parser.NewATNState(B__BlockEnd, parser.ATNBlockEnd, true) states[C_c] = parser.NewATNState(C_c, parser.ATNBasic, false) - states[C_common_0] = parser.NewATNState(C_common_0, parser.ATNBasic, false) - states[C_first] = parser.NewATNState(C_first, parser.ATNBasic, false) + states[C_COMMON_0] = parser.NewATNState(C_COMMON_0, parser.ATNBasic, false) + states[C_FIRST] = parser.NewATNState(C_FIRST, parser.ATNBasic, false) states[C__Basic_0] = parser.NewATNState(C__Basic_0, parser.ATNBasic, true) - states[C_common_1] = parser.NewATNState(C_common_1, parser.ATNBasic, false) - states[C_second] = parser.NewATNState(C_second, parser.ATNBasic, false) + states[C_COMMON_1] = parser.NewATNState(C_COMMON_1, parser.ATNBasic, false) + states[C_SECOND] = parser.NewATNState(C_SECOND, parser.ATNBasic, false) states[C__Basic_1] = parser.NewATNState(C__Basic_1, parser.ATNBasic, true) states[C__Basic_2] = parser.NewATNState(C__Basic_2, parser.ATNBasic, true).SetDecision(5) states[C__BlockEnd] = parser.NewATNState(C__BlockEnd, parser.ATNBlockEnd, true) @@ -342,11 +342,11 @@ func BuildATN() *parser.RuntimeATN { states[E_e] = parser.NewATNState(E_e, parser.ATNBasic, false) states[E__Basic_0] = parser.NewATNState(E__Basic_0, parser.ATNBasic, true) states[E__Basic_1] = parser.NewATNState(E__Basic_1, parser.ATNBasic, true) - states[DLong_common] = parser.NewATNState(DLong_common, parser.ATNBasic, false) - states[DLong_then] = parser.NewATNState(DLong_then, parser.ATNBasic, false) - states[DLong_long] = parser.NewATNState(DLong_long, parser.ATNBasic, false) + states[DLong_COMMON] = parser.NewATNState(DLong_COMMON, parser.ATNBasic, false) + states[DLong_THEN] = parser.NewATNState(DLong_THEN, parser.ATNBasic, false) + states[DLong_LONG] = parser.NewATNState(DLong_LONG, parser.ATNBasic, false) states[DLong__Basic] = parser.NewATNState(DLong__Basic, parser.ATNBasic, true) - states[DShort_common] = parser.NewATNState(DShort_common, parser.ATNBasic, false) + states[DShort_COMMON] = parser.NewATNState(DShort_COMMON, parser.ATNBasic, false) states[DShort__Basic] = parser.NewATNState(DShort__Basic, parser.ATNBasic, true) states[F_f] = parser.NewATNState(F_f, parser.ATNBasic, false) states[F__Basic_0] = parser.NewATNState(F__Basic_0, parser.ATNBasic, true) @@ -365,7 +365,7 @@ func BuildATN() *parser.RuntimeATN { states[I__Basic_0] = parser.NewATNState(I__Basic_0, parser.ATNBasic, true) states[I__Basic_1] = parser.NewATNState(I__Basic_1, parser.ATNBasic, true) states[MemberCall_Ref_ID_0] = parser.NewATNState(MemberCall_Ref_ID_0, parser.ATNBasic, false) - states[MemberCall_Dot] = parser.NewATNState(MemberCall_Dot, parser.ATNBasic, false) + states[MemberCall_DOT] = parser.NewATNState(MemberCall_DOT, parser.ATNBasic, false) states[MemberCall_Ref_ID_1] = parser.NewATNState(MemberCall_Ref_ID_1, parser.ATNBasic, false) states[MemberCall__Basic] = parser.NewATNState(MemberCall__Basic, parser.ATNBasic, true) states[MemberCall__LoopEntry] = parser.NewATNState(MemberCall__LoopEntry, parser.ATNLoopEntry, true).SetDecision(8) @@ -380,7 +380,7 @@ func BuildATN() *parser.RuntimeATN { states[J_j] = parser.NewATNState(J_j, parser.ATNBasic, false) states[J_Ref_ID] = parser.NewATNState(J_Ref_ID, parser.ATNBasic, false) states[J__Basic_0] = parser.NewATNState(J__Basic_0, parser.ATNBasic, true) - states[J_self] = parser.NewATNState(J_self, parser.ATNBasic, false) + states[J_SELF] = parser.NewATNState(J_SELF, parser.ATNBasic, false) states[J__Basic_1] = parser.NewATNState(J__Basic_1, parser.ATNBasic, true) states[J__Basic_2] = parser.NewATNState(J__Basic_2, parser.ATNBasic, true).SetDecision(10) states[J__BlockEnd] = parser.NewATNState(J__BlockEnd, parser.ATNBlockEnd, true) @@ -394,12 +394,12 @@ func BuildATN() *parser.RuntimeATN { states[K__Basic_2] = parser.NewATNState(K__Basic_2, parser.ATNBasic, true).SetDecision(11) states[K__BlockEnd] = parser.NewATNState(K__BlockEnd, parser.ATNBlockEnd, true) states[L_l] = parser.NewATNState(L_l, parser.ATNBasic, false) - states[L_optional] = parser.NewATNState(L_optional, parser.ATNBasic, false) - states[L_and] = parser.NewATNState(L_and, parser.ATNBasic, false) + states[L_OPTIONAL] = parser.NewATNState(L_OPTIONAL, parser.ATNBasic, false) + states[L_AND] = parser.NewATNState(L_AND, parser.ATNBasic, false) states[L__Basic_0] = parser.NewATNState(L__Basic_0, parser.ATNBasic, true) states[L__Basic_1] = parser.NewATNState(L__Basic_1, parser.ATNBasic, true).SetDecision(12) - states[L_then] = parser.NewATNState(L_then, parser.ATNBasic, false) - states[L_end] = parser.NewATNState(L_end, parser.ATNBasic, false) + states[L_THEN] = parser.NewATNState(L_THEN, parser.ATNBasic, false) + states[L_END] = parser.NewATNState(L_END, parser.ATNBasic, false) states[L__Basic_2] = parser.NewATNState(L__Basic_2, parser.ATNBasic, true) states[M_m] = parser.NewATNState(M_m, parser.ATNBasic, false) states[M__Basic_0] = parser.NewATNState(M__Basic_0, parser.ATNBasic, false) @@ -411,7 +411,7 @@ func BuildATN() *parser.RuntimeATN { states[O_Ref_ID] = parser.NewATNState(O_Ref_ID, parser.ATNBasic, false) states[O__Basic] = parser.NewATNState(O__Basic, parser.ATNBasic, true) states[FQN_ID_0] = parser.NewATNState(FQN_ID_0, parser.ATNBasic, false) - states[FQN_Dot] = parser.NewATNState(FQN_Dot, parser.ATNBasic, false) + states[FQN_DOT] = parser.NewATNState(FQN_DOT, parser.ATNBasic, false) states[FQN_ID_1] = parser.NewATNState(FQN_ID_1, parser.ATNBasic, false) states[FQN__Basic] = parser.NewATNState(FQN__Basic, parser.ATNBasic, true) states[FQN__LoopEntry] = parser.NewATNState(FQN__LoopEntry, parser.ATNLoopEntry, true).SetDecision(13) @@ -421,7 +421,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[Root__LoopEntry]), ) states[Declare__Start].AppendTransitions( - parser.NewEpsilonTransition(states[Declare_declare]), + parser.NewEpsilonTransition(states[Declare_DECLARE]), ) states[A__Start].AppendTransitions( parser.NewEpsilonTransition(states[A_a]), @@ -439,10 +439,10 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[E_e]), ) states[DLong__Start].AppendTransitions( - parser.NewEpsilonTransition(states[DLong_common]), + parser.NewEpsilonTransition(states[DLong_COMMON]), ) states[DShort__Start].AppendTransitions( - parser.NewEpsilonTransition(states[DShort_common]), + parser.NewEpsilonTransition(states[DShort_COMMON]), ) states[F__Start].AppendTransitions( parser.NewEpsilonTransition(states[F_f]), @@ -613,14 +613,14 @@ func BuildATN() *parser.RuntimeATN { states[Root__LoopBack].AppendTransitions( parser.NewEpsilonTransition(states[Root__LoopEntry]), ) - states[Declare_declare].AppendTransitions( - parser.NewAtomTransition(states[Declare__Basic_0], Keyword_declare, nil), + states[Declare_DECLARE].AppendTransitions( + parser.NewAtomTransition(states[Declare__Basic_0], Token_DECLARE, nil), ) states[Declare__Basic_0].AppendTransitions( parser.NewRuleTransition(states[FQN__Start], states[Declare__Basic_4], nil), ) - states[Declare_LeftBrace].AppendTransitions( - parser.NewAtomTransition(states[Declare__LoopEntry], Keyword_LeftBrace, nil), + states[Declare_LBRACE].AppendTransitions( + parser.NewAtomTransition(states[Declare__LoopEntry], Token_LBRACE, nil), ) states[Declare__Basic_1].AppendTransitions( parser.NewRuleTransition(states[Declare__Start], states[Declare__Basic_2], nil), @@ -633,26 +633,26 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[Declare__LoopEnd]), ) states[Declare__LoopEnd].AppendTransitions( - parser.NewEpsilonTransition(states[Declare_RightBrace]), + parser.NewEpsilonTransition(states[Declare_RBRACE]), ) states[Declare__LoopBack].AppendTransitions( parser.NewEpsilonTransition(states[Declare__LoopEntry]), ) - states[Declare_RightBrace].AppendTransitions( - parser.NewAtomTransition(states[Declare__Basic_3], Keyword_RightBrace, nil), + states[Declare_RBRACE].AppendTransitions( + parser.NewAtomTransition(states[Declare__Basic_3], Token_RBRACE, nil), ) states[Declare__Basic_3].AppendTransitions( parser.NewEpsilonTransition(states[Declare__Stop]), ) states[Declare__Basic_4].AppendTransitions( - parser.NewEpsilonTransition(states[Declare_LeftBrace]), + parser.NewEpsilonTransition(states[Declare_LBRACE]), parser.NewEpsilonTransition(states[Declare__Basic_3]), ) states[A_a].AppendTransitions( - parser.NewAtomTransition(states[A_first], Keyword_a, nil), + parser.NewAtomTransition(states[A_FIRST], Keyword_a, nil), ) - states[A_first].AppendTransitions( - parser.NewAtomTransition(states[A__Basic], Keyword_first, nil), + states[A_FIRST].AppendTransitions( + parser.NewAtomTransition(states[A__Basic], Token_FIRST, nil), ) states[A__Basic].AppendTransitions( parser.NewEpsilonTransition(states[A__Stop]), @@ -660,21 +660,21 @@ func BuildATN() *parser.RuntimeATN { states[B_b].AppendTransitions( parser.NewAtomTransition(states[B__Basic_2], Keyword_b, nil), ) - states[B_first].AppendTransitions( - parser.NewAtomTransition(states[B__Basic_0], Keyword_first, nil), + states[B_FIRST].AppendTransitions( + parser.NewAtomTransition(states[B__Basic_0], Token_FIRST, nil), ) states[B__Basic_0].AppendTransitions( parser.NewEpsilonTransition(states[B__BlockEnd]), ) - states[B_second].AppendTransitions( - parser.NewAtomTransition(states[B__Basic_1], Keyword_second, nil), + states[B_SECOND].AppendTransitions( + parser.NewAtomTransition(states[B__Basic_1], Token_SECOND, nil), ) states[B__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[B__BlockEnd]), ) states[B__Basic_2].AppendTransitions( - parser.NewEpsilonTransition(states[B_first]), - parser.NewEpsilonTransition(states[B_second]), + parser.NewEpsilonTransition(states[B_FIRST]), + parser.NewEpsilonTransition(states[B_SECOND]), ) states[B__BlockEnd].AppendTransitions( parser.NewEpsilonTransition(states[B__Stop]), @@ -682,27 +682,27 @@ func BuildATN() *parser.RuntimeATN { states[C_c].AppendTransitions( parser.NewAtomTransition(states[C__Basic_2], Keyword_c, nil), ) - states[C_common_0].AppendTransitions( - parser.NewAtomTransition(states[C_first], Keyword_common, nil), + states[C_COMMON_0].AppendTransitions( + parser.NewAtomTransition(states[C_FIRST], Token_COMMON, nil), ) - states[C_first].AppendTransitions( - parser.NewAtomTransition(states[C__Basic_0], Keyword_first, nil), + states[C_FIRST].AppendTransitions( + parser.NewAtomTransition(states[C__Basic_0], Token_FIRST, nil), ) states[C__Basic_0].AppendTransitions( parser.NewEpsilonTransition(states[C__BlockEnd]), ) - states[C_common_1].AppendTransitions( - parser.NewAtomTransition(states[C_second], Keyword_common, nil), + states[C_COMMON_1].AppendTransitions( + parser.NewAtomTransition(states[C_SECOND], Token_COMMON, nil), ) - states[C_second].AppendTransitions( - parser.NewAtomTransition(states[C__Basic_1], Keyword_second, nil), + states[C_SECOND].AppendTransitions( + parser.NewAtomTransition(states[C__Basic_1], Token_SECOND, nil), ) states[C__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[C__BlockEnd]), ) states[C__Basic_2].AppendTransitions( - parser.NewEpsilonTransition(states[C_common_0]), - parser.NewEpsilonTransition(states[C_common_1]), + parser.NewEpsilonTransition(states[C_COMMON_0]), + parser.NewEpsilonTransition(states[C_COMMON_1]), ) states[C__BlockEnd].AppendTransitions( parser.NewEpsilonTransition(states[C__Stop]), @@ -738,20 +738,20 @@ func BuildATN() *parser.RuntimeATN { states[E__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[E__Stop]), ) - states[DLong_common].AppendTransitions( - parser.NewAtomTransition(states[DLong_then], Keyword_common, nil), + states[DLong_COMMON].AppendTransitions( + parser.NewAtomTransition(states[DLong_THEN], Token_COMMON, nil), ) - states[DLong_then].AppendTransitions( - parser.NewAtomTransition(states[DLong_long], Keyword_then, nil), + states[DLong_THEN].AppendTransitions( + parser.NewAtomTransition(states[DLong_LONG], Token_THEN, nil), ) - states[DLong_long].AppendTransitions( - parser.NewAtomTransition(states[DLong__Basic], Keyword_long, nil), + states[DLong_LONG].AppendTransitions( + parser.NewAtomTransition(states[DLong__Basic], Token_LONG, nil), ) states[DLong__Basic].AppendTransitions( parser.NewEpsilonTransition(states[DLong__Stop]), ) - states[DShort_common].AppendTransitions( - parser.NewAtomTransition(states[DShort__Basic], Keyword_common, nil), + states[DShort_COMMON].AppendTransitions( + parser.NewAtomTransition(states[DShort__Basic], Token_COMMON, nil), ) states[DShort__Basic].AppendTransitions( parser.NewEpsilonTransition(states[DShort__Stop]), @@ -808,8 +808,8 @@ func BuildATN() *parser.RuntimeATN { states[MemberCall_Ref_ID_0].AppendTransitions( parser.NewAtomTransition(states[MemberCall__LoopEntry], Token_ID, &parser.CompletionHint{Field: "MemberCall.Ref"}), ) - states[MemberCall_Dot].AppendTransitions( - parser.NewAtomTransition(states[MemberCall_Ref_ID_1], Keyword_Dot, nil), + states[MemberCall_DOT].AppendTransitions( + parser.NewAtomTransition(states[MemberCall_Ref_ID_1], Token_DOT, nil), ) states[MemberCall_Ref_ID_1].AppendTransitions( parser.NewAtomTransition(states[MemberCall__Basic], Token_ID, &parser.CompletionHint{Field: "MemberCall.Ref"}), @@ -818,7 +818,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[MemberCall__LoopBack]), ) states[MemberCall__LoopEntry].AppendTransitions( - parser.NewEpsilonTransition(states[MemberCall_Dot]), + parser.NewEpsilonTransition(states[MemberCall_DOT]), parser.NewEpsilonTransition(states[MemberCall__LoopEnd]), ) states[MemberCall__LoopEnd].AppendTransitions( @@ -855,15 +855,15 @@ func BuildATN() *parser.RuntimeATN { states[J__Basic_0].AppendTransitions( parser.NewEpsilonTransition(states[J__BlockEnd]), ) - states[J_self].AppendTransitions( - parser.NewAtomTransition(states[J__Basic_1], Keyword_self, nil), + states[J_SELF].AppendTransitions( + parser.NewAtomTransition(states[J__Basic_1], Token_SELF, nil), ) states[J__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[J__BlockEnd]), ) states[J__Basic_2].AppendTransitions( parser.NewEpsilonTransition(states[J_Ref_ID]), - parser.NewEpsilonTransition(states[J_self]), + parser.NewEpsilonTransition(states[J_SELF]), ) states[J__BlockEnd].AppendTransitions( parser.NewEpsilonTransition(states[J__Stop]), @@ -899,24 +899,24 @@ func BuildATN() *parser.RuntimeATN { states[L_l].AppendTransitions( parser.NewAtomTransition(states[L__Basic_1], Keyword_l, nil), ) - states[L_optional].AppendTransitions( - parser.NewAtomTransition(states[L_and], Keyword_optional, nil), + states[L_OPTIONAL].AppendTransitions( + parser.NewAtomTransition(states[L_AND], Token_OPTIONAL, nil), ) - states[L_and].AppendTransitions( - parser.NewAtomTransition(states[L__Basic_0], Keyword_and, nil), + states[L_AND].AppendTransitions( + parser.NewAtomTransition(states[L__Basic_0], Token_AND, nil), ) states[L__Basic_0].AppendTransitions( - parser.NewEpsilonTransition(states[L_then]), + parser.NewEpsilonTransition(states[L_THEN]), ) states[L__Basic_1].AppendTransitions( - parser.NewEpsilonTransition(states[L_optional]), + parser.NewEpsilonTransition(states[L_OPTIONAL]), parser.NewEpsilonTransition(states[L__Basic_0]), ) - states[L_then].AppendTransitions( - parser.NewAtomTransition(states[L_end], Keyword_then, nil), + states[L_THEN].AppendTransitions( + parser.NewAtomTransition(states[L_END], Token_THEN, nil), ) - states[L_end].AppendTransitions( - parser.NewAtomTransition(states[L__Basic_2], Keyword_end, nil), + states[L_END].AppendTransitions( + parser.NewAtomTransition(states[L__Basic_2], Token_END, nil), ) states[L__Basic_2].AppendTransitions( parser.NewEpsilonTransition(states[L__Stop]), @@ -925,7 +925,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[M__Basic_0], Keyword_m, nil), ) states[M__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[M__Basic_1], Token_SomeTokenGroup, nil), + parser.NewAtomTransition(states[M__Basic_1], TokenGroup_SomeTokenGroup, nil), ) states[M__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[M__Stop]), @@ -934,7 +934,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[N__Basic_0], Keyword_n, nil), ) states[N__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[N__Basic_1], Token_SomeTokenGroup, &parser.CompletionHint{Field: "N.Ref"}), + parser.NewAtomTransition(states[N__Basic_1], TokenGroup_SomeTokenGroup, &parser.CompletionHint{Field: "N.Ref"}), ) states[N__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[N__Stop]), @@ -951,8 +951,8 @@ func BuildATN() *parser.RuntimeATN { states[FQN_ID_0].AppendTransitions( parser.NewAtomTransition(states[FQN__LoopEntry], Token_ID, nil), ) - states[FQN_Dot].AppendTransitions( - parser.NewAtomTransition(states[FQN_ID_1], Keyword_Dot, nil), + states[FQN_DOT].AppendTransitions( + parser.NewAtomTransition(states[FQN_ID_1], Token_DOT, nil), ) states[FQN_ID_1].AppendTransitions( parser.NewAtomTransition(states[FQN__Basic], Token_ID, nil), @@ -961,7 +961,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[FQN__LoopBack]), ) states[FQN__LoopEntry].AppendTransitions( - parser.NewEpsilonTransition(states[FQN_Dot]), + parser.NewEpsilonTransition(states[FQN_DOT]), parser.NewEpsilonTransition(states[FQN__LoopEnd]), ) states[FQN__LoopEnd].AppendTransitions( diff --git a/internal/languages/completion/completion.fb b/internal/languages/completion/completion.fb index 5327d5dc..dd3174f7 100644 --- a/internal/languages/completion/completion.fb +++ b/internal/languages/completion/completion.fb @@ -14,18 +14,18 @@ interface Declare extends Obj { Children []Declare } -Declare: "declare" Name=FQN ("{" Children+=Declare* "}")?; +Declare: DECLARE Name=FQN (LBRACE Children+=Declare* RBRACE)?; -composite FQN: ID ("." ID)*; +composite FQN: ID (DOT ID)*; // straight-line continuation -A returns Obj: "a" "first"; +A returns Obj: "a" FIRST; // flat alternative - both branches must stay live post-commit -B returns Obj: "b" ("first" | "second"); +B returns Obj: "b" (FIRST | SECOND); // shared-prefix alternative - keep both alts past the prefix -C returns Obj: "c" ("common" "first" | "common" "second"); +C returns Obj: "c" (COMMON FIRST | COMMON SECOND); // rule-call alternative with shared prefix (DLong/DShort) D returns Obj: "d" (DLong | DShort); @@ -37,8 +37,8 @@ interface E extends Obj { // cross-reference via FQN composite E: "e" Ref=[Declare:FQN]; -DLong returns Obj: "common" "then" "long"; -DShort returns Obj: "common"; +DLong returns Obj: COMMON THEN LONG; +DShort returns Obj: COMMON; interface F extends Obj { Items []FItem @@ -76,7 +76,7 @@ H: "h" Member=MemberCall; // ensures that synthetic references can properly complete I returns H: "i" Member=MemberCallNoDot; -MemberCall: Ref=[Declare:ID] ({MemberCall.Previous=current} "." Ref=[Declare:ID])*; +MemberCall: Ref=[Declare:ID] ({MemberCall.Previous=current} DOT Ref=[Declare:ID])*; MemberCallNoDot returns MemberCall: Ref=[Declare:ID] ({MemberCall.Previous=current} Ref=[Declare:ID])*; @@ -85,7 +85,7 @@ interface J extends Obj { } // cross-reference and keyword alternative at the same position -J: "j" (Ref=[Declare:ID] | "self"); +J: "j" (Ref=[Declare:ID] | SELF); interface K extends Obj { Ref1 *Declare @@ -96,7 +96,7 @@ interface K extends Obj { K: "k" (Ref1=[Declare:ID] "x" | Ref2=[Declare:ID] "y"); // fully optional group in front of mandatory content -L returns Obj: "l" ("optional" "and")? "then" "end"; +L returns Obj: "l" (OPTIONAL AND)? THEN END; token group SomeTokenGroup { ID "a" "b" @@ -119,5 +119,29 @@ interface O extends Obj { // action used to change type O returns Obj: {O} "o" Ref=[Declare:ID]; -hidden token WS: /[ \t\r\n]+/; -token ID: /\w[\w0-9]*/; +token DECLARE: "declare" +token FIRST: "first" +token SECOND: "second" +token LONG: "long" +token SELF: "self" +token COMMON: "common" +token THEN: "then" +token LBRACE: "{" +token RBRACE: "}" +token DOT: "." +token OPTIONAL: "optional" +token AND: "and" +token END: "end" + +token WS: /[ \t\r\n]+/ +token ID: /\w[\w0-9]*/ + +token mode default { + keywords /^[a-z]+$/ + LBRACE + RBRACE + DOT + ID + + hidden WS +} diff --git a/internal/languages/completion/completion_engine_test.go b/internal/languages/completion/completion_engine_test.go index 7c927172..389a452e 100644 --- a/internal/languages/completion/completion_engine_test.go +++ b/internal/languages/completion/completion_engine_test.go @@ -241,6 +241,7 @@ func TestCompletion_PartialKeyword(t *testing.T) { declare := itemWithLabel(items, "declare") if declare == nil { t.Fatalf("expected 'declare'; got %v", itemLabels(items)) + return } if declare.TextEdit == nil { t.Errorf("expected REPLACE TextEdit on partial-keyword item; got nil") @@ -253,6 +254,7 @@ func TestCompletion_InsideKeyword(t *testing.T) { declare := itemWithLabel(items, "declare") if declare == nil { t.Fatalf("expected 'declare'; got %v", itemLabels(items)) + return } if declare.TextEdit == nil { t.Errorf("expected REPLACE TextEdit on full-keyword item; got nil") @@ -389,10 +391,10 @@ func TestCompletion_AfterH_MemberCall(t *testing.T) { if dot == nil || gamma == nil { t.Fatalf("expected '.' and 'gamma'; got %v", itemLabels(items)) } - if dot.TextEdit == nil { + if dot != nil && dot.TextEdit == nil { t.Errorf("expected '.' to carry a REPLACE TextEdit") } - if gamma.TextEdit != nil { + if gamma != nil && gamma.TextEdit != nil { t.Errorf("expected 'gamma' to be INSERT-shaped (no TextEdit); got %+v", gamma.TextEdit) } } @@ -658,6 +660,7 @@ func TestCompletion_ContributorTokenDocs(t *testing.T) { declare := itemWithLabel(items, "declare") if declare == nil { t.Fatalf("expected 'declare'; got %v", itemLabels(items)) + return } if declare.Documentation == nil { t.Errorf("expected Documentation set; got nil") @@ -788,6 +791,7 @@ func TestCompletion_ContributorPostProcess(t *testing.T) { c := itemWithLabel(items, "c") if c == nil { t.Fatalf("expected 'c' keyword; got %v", itemLabels(items)) + return } if !strings.HasPrefix(c.SortText, "zzz-") { t.Errorf("expected rewritten SortText; got %q", c.SortText) diff --git a/internal/languages/completion/completion_parser_gen.go b/internal/languages/completion/completion_parser_gen.go index bad67429..e5d8a300 100644 --- a/internal/languages/completion/completion_parser_gen.go +++ b/internal/languages/completion/completion_parser_gen.go @@ -157,7 +157,7 @@ func (p *CompletionParser) ParseDeclare() { p.cp.EnterRule("Declare", Declare__Start) defer p.cp.ExitRule() { - p.state.Consume(Keyword_declare) + p.state.Consume(Token_DECLARE) } { p.cp.MarkAssignment("Name") @@ -170,7 +170,7 @@ func (p *CompletionParser) ParseDeclare() { p.state.Sync(Declare__Basic_4) if p.lookahead.DeclareOptional(p.state) { { - p.state.Consume(Keyword_LeftBrace) + p.state.Consume(Token_LBRACE) } { p.cp.RecordSnapshot(Declare__LoopEntry) @@ -186,7 +186,7 @@ func (p *CompletionParser) ParseDeclare() { } } { - p.state.Consume(Keyword_RightBrace) + p.state.Consume(Token_RBRACE) } } } @@ -198,7 +198,7 @@ func (p *CompletionParser) ParseA() { p.state.Consume(Keyword_a) } { - p.state.Consume(Keyword_first) + p.state.Consume(Token_FIRST) } } @@ -211,11 +211,11 @@ func (p *CompletionParser) ParseB() { switch prediction, failure := p.lookahead.BAlternatives(p.state); prediction { case 0: { - p.state.Consume(Keyword_first) + p.state.Consume(Token_FIRST) } case 1: { - p.state.Consume(Keyword_second) + p.state.Consume(Token_SECOND) } default: p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) @@ -231,17 +231,17 @@ func (p *CompletionParser) ParseC() { switch prediction, failure := p.lookahead.CAlternatives(p.state); prediction { case 0: { - p.state.Consume(Keyword_common) + p.state.Consume(Token_COMMON) } { - p.state.Consume(Keyword_first) + p.state.Consume(Token_FIRST) } case 1: { - p.state.Consume(Keyword_common) + p.state.Consume(Token_COMMON) } { - p.state.Consume(Keyword_second) + p.state.Consume(Token_SECOND) } default: p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) @@ -291,13 +291,13 @@ func (p *CompletionParser) ParseDLong() { p.cp.EnterRule("DLong", DLong__Start) defer p.cp.ExitRule() { - p.state.Consume(Keyword_common) + p.state.Consume(Token_COMMON) } { - p.state.Consume(Keyword_then) + p.state.Consume(Token_THEN) } { - p.state.Consume(Keyword_long) + p.state.Consume(Token_LONG) } } @@ -305,7 +305,7 @@ func (p *CompletionParser) ParseDShort() { p.cp.EnterRule("DShort", DShort__Start) defer p.cp.ExitRule() { - p.state.Consume(Keyword_common) + p.state.Consume(Token_COMMON) } } @@ -395,7 +395,7 @@ func (p *CompletionParser) ParseMemberCall() { p.state.Sync(MemberCall__LoopEntry) for p.lookahead.MemberCallLoop(p.state) { { - p.state.Consume(Keyword_Dot) + p.state.Consume(Token_DOT) } { p.cp.MarkAssignment("Ref") @@ -443,7 +443,7 @@ func (p *CompletionParser) ParseJ() { } case 1: { - p.state.Consume(Keyword_self) + p.state.Consume(Token_SELF) } default: p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) @@ -490,17 +490,17 @@ func (p *CompletionParser) ParseL() { p.state.Sync(L__Basic_1) if p.lookahead.LOptional(p.state) { { - p.state.Consume(Keyword_optional) + p.state.Consume(Token_OPTIONAL) } { - p.state.Consume(Keyword_and) + p.state.Consume(Token_AND) } } { - p.state.Consume(Keyword_then) + p.state.Consume(Token_THEN) } { - p.state.Consume(Keyword_end) + p.state.Consume(Token_END) } } @@ -511,7 +511,7 @@ func (p *CompletionParser) ParseM() { p.state.Consume(Keyword_m) } { - p.state.Consume(Token_SomeTokenGroup) + p.state.Consume(TokenGroup_SomeTokenGroup) } } @@ -523,7 +523,7 @@ func (p *CompletionParser) ParseN() { } { p.cp.MarkAssignment("Ref") - p.state.Consume(Token_SomeTokenGroup) + p.state.Consume(TokenGroup_SomeTokenGroup) p.cp.ClearAssignment() } } @@ -551,7 +551,7 @@ func (p *CompletionParser) ParseFQN() { p.state.Sync(FQN__LoopEntry) for p.lookahead.FQNLoop(p.state) { { - p.state.Consume(Keyword_Dot) + p.state.Consume(Token_DOT) } { p.state.Consume(Token_ID) diff --git a/internal/languages/completion/lexer_gen.go b/internal/languages/completion/lexer_gen.go index 3bf081e9..a396ea83 100644 --- a/internal/languages/completion/lexer_gen.go +++ b/internal/languages/completion/lexer_gen.go @@ -15,10 +15,7 @@ var Keyword_Dot = core.NewTokenType( Keyword_Dot_Idx, ".", ".", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ".") { return 1 @@ -34,10 +31,7 @@ var Keyword_a = core.NewTokenType( Keyword_a_Idx, "a", "a", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "a") { return 1 @@ -53,10 +47,7 @@ var Keyword_and = core.NewTokenType( Keyword_and_Idx, "and", "and", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "and") { return 3 @@ -72,10 +63,7 @@ var Keyword_b = core.NewTokenType( Keyword_b_Idx, "b", "b", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "b") { return 1 @@ -91,10 +79,7 @@ var Keyword_c = core.NewTokenType( Keyword_c_Idx, "c", "c", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "c") { return 1 @@ -110,10 +95,7 @@ var Keyword_common = core.NewTokenType( Keyword_common_Idx, "common", "common", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "common") { return 6 @@ -129,10 +111,7 @@ var Keyword_d = core.NewTokenType( Keyword_d_Idx, "d", "d", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "d") { return 1 @@ -148,10 +127,7 @@ var Keyword_declare = core.NewTokenType( Keyword_declare_Idx, "declare", "declare", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "declare") { return 7 @@ -167,10 +143,7 @@ var Keyword_e = core.NewTokenType( Keyword_e_Idx, "e", "e", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "e") { return 1 @@ -186,10 +159,7 @@ var Keyword_end = core.NewTokenType( Keyword_end_Idx, "end", "end", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "end") { return 3 @@ -205,10 +175,7 @@ var Keyword_f = core.NewTokenType( Keyword_f_Idx, "f", "f", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "f") { return 1 @@ -224,10 +191,7 @@ var Keyword_first = core.NewTokenType( Keyword_first_Idx, "first", "first", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "first") { return 5 @@ -243,10 +207,7 @@ var Keyword_g = core.NewTokenType( Keyword_g_Idx, "g", "g", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "g") { return 1 @@ -262,10 +223,7 @@ var Keyword_h = core.NewTokenType( Keyword_h_Idx, "h", "h", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "h") { return 1 @@ -281,10 +239,7 @@ var Keyword_i = core.NewTokenType( Keyword_i_Idx, "i", "i", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "i") { return 1 @@ -300,10 +255,7 @@ var Keyword_j = core.NewTokenType( Keyword_j_Idx, "j", "j", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "j") { return 1 @@ -319,10 +271,7 @@ var Keyword_k = core.NewTokenType( Keyword_k_Idx, "k", "k", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "k") { return 1 @@ -338,10 +287,7 @@ var Keyword_l = core.NewTokenType( Keyword_l_Idx, "l", "l", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "l") { return 1 @@ -357,10 +303,7 @@ var Keyword_long = core.NewTokenType( Keyword_long_Idx, "long", "long", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "long") { return 4 @@ -376,10 +319,7 @@ var Keyword_m = core.NewTokenType( Keyword_m_Idx, "m", "m", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "m") { return 1 @@ -395,10 +335,7 @@ var Keyword_n = core.NewTokenType( Keyword_n_Idx, "n", "n", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "n") { return 1 @@ -414,10 +351,7 @@ var Keyword_o = core.NewTokenType( Keyword_o_Idx, "o", "o", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "o") { return 1 @@ -433,10 +367,7 @@ var Keyword_optional = core.NewTokenType( Keyword_optional_Idx, "optional", "optional", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "optional") { return 8 @@ -452,10 +383,7 @@ var Keyword_second = core.NewTokenType( Keyword_second_Idx, "second", "second", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "second") { return 6 @@ -471,10 +399,7 @@ var Keyword_self = core.NewTokenType( Keyword_self_Idx, "self", "self", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "self") { return 4 @@ -490,10 +415,7 @@ var Keyword_then = core.NewTokenType( Keyword_then_Idx, "then", "then", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "then") { return 4 @@ -509,10 +431,7 @@ var Keyword_x = core.NewTokenType( Keyword_x_Idx, "x", "x", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "x") { return 1 @@ -528,10 +447,7 @@ var Keyword_y = core.NewTokenType( Keyword_y_Idx, "y", "y", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "y") { return 1 @@ -547,10 +463,7 @@ var Keyword_LeftBrace = core.NewTokenType( Keyword_LeftBrace_Idx, "{", "{", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "{") { return 1 @@ -566,10 +479,7 @@ var Keyword_RightBrace = core.NewTokenType( Keyword_RightBrace_Idx, "}", "}", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "}") { return 1 @@ -579,16 +489,65 @@ var Keyword_RightBrace = core.NewTokenType( []rune{'}'}, ) +const Token_DECLARE_Idx = Keyword_declare_Idx + +var Token_DECLARE = Keyword_declare + +const Token_FIRST_Idx = Keyword_first_Idx + +var Token_FIRST = Keyword_first + +const Token_SECOND_Idx = Keyword_second_Idx + +var Token_SECOND = Keyword_second + +const Token_LONG_Idx = Keyword_long_Idx + +var Token_LONG = Keyword_long + +const Token_SELF_Idx = Keyword_self_Idx + +var Token_SELF = Keyword_self + +const Token_COMMON_Idx = Keyword_common_Idx + +var Token_COMMON = Keyword_common + +const Token_THEN_Idx = Keyword_then_Idx + +var Token_THEN = Keyword_then + +const Token_LBRACE_Idx = Keyword_LeftBrace_Idx + +var Token_LBRACE = Keyword_LeftBrace + +const Token_RBRACE_Idx = Keyword_RightBrace_Idx + +var Token_RBRACE = Keyword_RightBrace + +const Token_DOT_Idx = Keyword_Dot_Idx + +var Token_DOT = Keyword_Dot + +const Token_OPTIONAL_Idx = Keyword_optional_Idx + +var Token_OPTIONAL = Keyword_optional + +const Token_AND_Idx = Keyword_and_Idx + +var Token_AND = Keyword_and + +const Token_END_Idx = Keyword_end_Idx + +var Token_END = Keyword_end + const Token_WS_Idx = 31 var Token_WS = core.NewTokenType( Token_WS_Idx, "WS", "WS", - core.SkippedGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -659,10 +618,7 @@ var Token_ID = core.NewTokenType( Token_ID_Idx, "ID", "ID", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -727,10 +683,10 @@ var Token_ID_Accepting = [2]bool{ 1: true, } -const Token_SomeTokenGroup_Idx = 33 +const TokenGroup_SomeTokenGroup_Idx = 33 -var Token_SomeTokenGroup = core.NewTokenGroup( - Token_SomeTokenGroup_Idx, +var TokenGroup_SomeTokenGroup = core.NewTokenGroup( + TokenGroup_SomeTokenGroup_Idx, "SomeTokenGroup", "SomeTokenGroup", []*core.TokenType{ @@ -740,39 +696,45 @@ var Token_SomeTokenGroup = core.NewTokenGroup( }, ) +const ( + TokenMode_default = 0 +) + func NewLexer() lexer.Lexer { - return lexer.NewDefaultLexer( - Keyword_Dot, - Keyword_a, - Keyword_and, - Keyword_b, - Keyword_c, - Keyword_common, - Keyword_d, - Keyword_declare, - Keyword_e, - Keyword_end, - Keyword_f, - Keyword_first, - Keyword_g, - Keyword_h, - Keyword_i, - Keyword_j, - Keyword_k, - Keyword_l, - Keyword_long, - Keyword_m, - Keyword_n, - Keyword_o, - Keyword_optional, - Keyword_second, - Keyword_self, - Keyword_then, - Keyword_x, - Keyword_y, - Keyword_LeftBrace, - Keyword_RightBrace, - Token_WS, - Token_ID, + modes := make([]*lexer.TokenMode, 1) + modes[TokenMode_default] = lexer.NewTokenMode("default", + lexer.UseTokenType(Keyword_a), + lexer.UseTokenType(Token_AND), + lexer.UseTokenType(Keyword_b), + lexer.UseTokenType(Keyword_c), + lexer.UseTokenType(Token_COMMON), + lexer.UseTokenType(Keyword_d), + lexer.UseTokenType(Token_DECLARE), + lexer.UseTokenType(Keyword_e), + lexer.UseTokenType(Token_END), + lexer.UseTokenType(Keyword_f), + lexer.UseTokenType(Token_FIRST), + lexer.UseTokenType(Keyword_g), + lexer.UseTokenType(Keyword_h), + lexer.UseTokenType(Keyword_i), + lexer.UseTokenType(Keyword_j), + lexer.UseTokenType(Keyword_k), + lexer.UseTokenType(Keyword_l), + lexer.UseTokenType(Token_LONG), + lexer.UseTokenType(Keyword_m), + lexer.UseTokenType(Keyword_n), + lexer.UseTokenType(Keyword_o), + lexer.UseTokenType(Token_OPTIONAL), + lexer.UseTokenType(Token_SECOND), + lexer.UseTokenType(Token_SELF), + lexer.UseTokenType(Token_THEN), + lexer.UseTokenType(Keyword_x), + lexer.UseTokenType(Keyword_y), + lexer.UseTokenType(Token_LBRACE), + lexer.UseTokenType(Token_RBRACE), + lexer.UseTokenType(Token_DOT), + lexer.UseTokenType(Token_ID), + lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier), ) + return lexer.NewDefaultLexer(TokenMode_default, modes...) } diff --git a/internal/languages/completion/parser_gen.go b/internal/languages/completion/parser_gen.go index 686d7b5e..ce0ade9c 100644 --- a/internal/languages/completion/parser_gen.go +++ b/internal/languages/completion/parser_gen.go @@ -169,8 +169,8 @@ func (p *Parser) ParseDeclare() Declare { current.SetTextRangeStart(p.state.LA(1).Range.Start) { { - token := p.state.Consume(Keyword_declare) - core.AssignToken(current, token, Declare_declare) + token := p.state.Consume(Token_DECLARE) + core.AssignToken(current, token, Declare_DECLARE) } { result := core.NewCompositeNode() @@ -186,8 +186,8 @@ func (p *Parser) ParseDeclare() Declare { p.state.Sync(Declare__Basic_4) if p.lookahead.DeclareOptional(p.state) { { - token := p.state.Consume(Keyword_LeftBrace) - core.AssignToken(current, token, Declare_LeftBrace) + token := p.state.Consume(Token_LBRACE) + core.AssignToken(current, token, Declare_LBRACE) } { p.state.Sync(Declare__LoopEntry) @@ -202,8 +202,8 @@ func (p *Parser) ParseDeclare() Declare { } } { - token := p.state.Consume(Keyword_RightBrace) - core.AssignToken(current, token, Declare_RightBrace) + token := p.state.Consume(Token_RBRACE) + core.AssignToken(current, token, Declare_RBRACE) } } } @@ -220,8 +220,8 @@ func (p *Parser) ParseA() Obj { core.AssignToken(current, token, A_a) } { - token := p.state.Consume(Keyword_first) - core.AssignToken(current, token, A_first) + token := p.state.Consume(Token_FIRST) + core.AssignToken(current, token, A_FIRST) } } current.SetTextRangeEnd(p.state.LA(0).Range.End) @@ -239,13 +239,13 @@ func (p *Parser) ParseB() Obj { switch prediction, failure := p.lookahead.BAlternatives(p.state); prediction { case 0: { - token := p.state.Consume(Keyword_first) - core.AssignToken(current, token, B_first) + token := p.state.Consume(Token_FIRST) + core.AssignToken(current, token, B_FIRST) } case 1: { - token := p.state.Consume(Keyword_second) - core.AssignToken(current, token, B_second) + token := p.state.Consume(Token_SECOND) + core.AssignToken(current, token, B_SECOND) } default: p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) @@ -266,21 +266,21 @@ func (p *Parser) ParseC() Obj { switch prediction, failure := p.lookahead.CAlternatives(p.state); prediction { case 0: { - token := p.state.Consume(Keyword_common) - core.AssignToken(current, token, C_common_0) + token := p.state.Consume(Token_COMMON) + core.AssignToken(current, token, C_COMMON_0) } { - token := p.state.Consume(Keyword_first) - core.AssignToken(current, token, C_first) + token := p.state.Consume(Token_FIRST) + core.AssignToken(current, token, C_FIRST) } case 1: { - token := p.state.Consume(Keyword_common) - core.AssignToken(current, token, C_common_1) + token := p.state.Consume(Token_COMMON) + core.AssignToken(current, token, C_COMMON_1) } { - token := p.state.Consume(Keyword_second) - core.AssignToken(current, token, C_second) + token := p.state.Consume(Token_SECOND) + core.AssignToken(current, token, C_SECOND) } default: p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) @@ -352,16 +352,16 @@ func (p *Parser) ParseDLong() Obj { current.SetTextRangeStart(p.state.LA(1).Range.Start) { { - token := p.state.Consume(Keyword_common) - core.AssignToken(current, token, DLong_common) + token := p.state.Consume(Token_COMMON) + core.AssignToken(current, token, DLong_COMMON) } { - token := p.state.Consume(Keyword_then) - core.AssignToken(current, token, DLong_then) + token := p.state.Consume(Token_THEN) + core.AssignToken(current, token, DLong_THEN) } { - token := p.state.Consume(Keyword_long) - core.AssignToken(current, token, DLong_long) + token := p.state.Consume(Token_LONG) + core.AssignToken(current, token, DLong_LONG) } } current.SetTextRangeEnd(p.state.LA(0).Range.End) @@ -373,8 +373,8 @@ func (p *Parser) ParseDShort() Obj { current.SetTextRangeStart(p.state.LA(1).Range.Start) { { - token := p.state.Consume(Keyword_common) - core.AssignToken(current, token, DShort_common) + token := p.state.Consume(Token_COMMON) + core.AssignToken(current, token, DShort_COMMON) } } current.SetTextRangeEnd(p.state.LA(0).Range.End) @@ -509,8 +509,8 @@ func (p *Parser) ParseMemberCall() MemberCall { } current := current.(MemberCall) { - token := p.state.Consume(Keyword_Dot) - core.AssignToken(current, token, MemberCall_Dot) + token := p.state.Consume(Token_DOT) + core.AssignToken(current, token, MemberCall_DOT) } { token := p.state.Consume(Token_ID) @@ -580,8 +580,8 @@ func (p *Parser) ParseJ() J { } case 1: { - token := p.state.Consume(Keyword_self) - core.AssignToken(current, token, J_self) + token := p.state.Consume(Token_SELF) + core.AssignToken(current, token, J_SELF) } default: p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) @@ -643,21 +643,21 @@ func (p *Parser) ParseL() Obj { p.state.Sync(L__Basic_1) if p.lookahead.LOptional(p.state) { { - token := p.state.Consume(Keyword_optional) - core.AssignToken(current, token, L_optional) + token := p.state.Consume(Token_OPTIONAL) + core.AssignToken(current, token, L_OPTIONAL) } { - token := p.state.Consume(Keyword_and) - core.AssignToken(current, token, L_and) + token := p.state.Consume(Token_AND) + core.AssignToken(current, token, L_AND) } } { - token := p.state.Consume(Keyword_then) - core.AssignToken(current, token, L_then) + token := p.state.Consume(Token_THEN) + core.AssignToken(current, token, L_THEN) } { - token := p.state.Consume(Keyword_end) - core.AssignToken(current, token, L_end) + token := p.state.Consume(Token_END) + core.AssignToken(current, token, L_END) } } current.SetTextRangeEnd(p.state.LA(0).Range.End) @@ -673,7 +673,7 @@ func (p *Parser) ParseM() Obj { core.AssignToken(current, token, M_m) } { - token := p.state.Consume(Token_SomeTokenGroup) + token := p.state.Consume(TokenGroup_SomeTokenGroup) core.AssignToken(current, token, M__Basic_0) } } @@ -690,7 +690,7 @@ func (p *Parser) ParseN() N { core.AssignToken(current, token, N_n) } { - token := p.state.Consume(Token_SomeTokenGroup) + token := p.state.Consume(TokenGroup_SomeTokenGroup) core.AssignToken(current, token, N__Basic_0) if token != nil { current.SetRef(p.referencesConstructor.NRef(current, token)) @@ -735,8 +735,8 @@ func (p *Parser) ParseFQN(current core.CompositeNode) { p.state.Sync(FQN__LoopEntry) for p.lookahead.FQNLoop(p.state) { { - token := p.state.Consume(Keyword_Dot) - core.AssignToken(current, token, FQN_Dot) + token := p.state.Consume(Token_DOT) + core.AssignToken(current, token, FQN_DOT) } { token := p.state.Consume(Token_ID) diff --git a/internal/languages/completion/parser_lookahead_gen.go b/internal/languages/completion/parser_lookahead_gen.go index 91e2f712..72e459d2 100644 --- a/internal/languages/completion/parser_lookahead_gen.go +++ b/internal/languages/completion/parser_lookahead_gen.go @@ -14,17 +14,17 @@ const ( ) var BAlternatives = parser.LL1Lookahead{ - Types: []*core.TokenType{Keyword_first, Keyword_second}, + Types: []*core.TokenType{Token_FIRST, Token_SECOND}, Lookup: []int{12: 1, 24: 2}, } var JAlternatives = parser.LL1Lookahead{ - Types: []*core.TokenType{Token_ID, Keyword_self}, + Types: []*core.TokenType{Token_ID, Token_SELF}, Lookup: []int{25: 2, 32: 1}, } var RootObjectsAlternatives = parser.LL1Lookahead{ - Types: []*core.TokenType{Keyword_declare, Keyword_a, Keyword_b, Keyword_c, Keyword_d, Keyword_e, Keyword_f, Keyword_g, Keyword_h, Keyword_i, Keyword_j, Keyword_k, Keyword_l, Keyword_m, Keyword_n, Keyword_o}, + Types: []*core.TokenType{Token_DECLARE, Keyword_a, Keyword_b, Keyword_c, Keyword_d, Keyword_e, Keyword_f, Keyword_g, Keyword_h, Keyword_i, Keyword_j, Keyword_k, Keyword_l, Keyword_m, Keyword_n, Keyword_o}, Lookup: []int{2: 2, 4: 3, 5: 4, 7: 5, 8: 1, 9: 6, 11: 7, 13: 8, 14: 9, 15: 10, 16: 11, 17: 12, 18: 13, 20: 14, 21: 15, 22: 16}, } @@ -73,11 +73,11 @@ func (l *DefaultCompletionParserLookahead) DAlternatives(state *parser.ParserSta } func (l *DefaultCompletionParserLookahead) DeclareChildrenLoop(state *parser.ParserState) bool { - return state.LA(1).Type == Keyword_declare + return state.LA(1).Type == Token_DECLARE } func (l *DefaultCompletionParserLookahead) DeclareOptional(state *parser.ParserState) bool { - return state.LA(1).Type == Keyword_LeftBrace + return state.LA(1).Type == Token_LBRACE } func (l *DefaultCompletionParserLookahead) FItemsLoop(state *parser.ParserState) bool { @@ -85,7 +85,7 @@ func (l *DefaultCompletionParserLookahead) FItemsLoop(state *parser.ParserState) } func (l *DefaultCompletionParserLookahead) FQNLoop(state *parser.ParserState) bool { - return state.LA(1).Type == Keyword_Dot + return state.LA(1).Type == Token_DOT } func (l *DefaultCompletionParserLookahead) JAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { @@ -97,11 +97,11 @@ func (l *DefaultCompletionParserLookahead) KAlternatives(state *parser.ParserSta } func (l *DefaultCompletionParserLookahead) LOptional(state *parser.ParserState) bool { - return state.LA(1).Type == Keyword_optional + return state.LA(1).Type == Token_OPTIONAL } func (l *DefaultCompletionParserLookahead) MemberCallLoop(state *parser.ParserState) bool { - return state.LA(1).Type == Keyword_Dot + return state.LA(1).Type == Token_DOT } func (l *DefaultCompletionParserLookahead) MemberCallNoDotLoop(state *parser.ParserState) bool { diff --git a/internal/languages/lookahead/lexer_gen.go b/internal/languages/lookahead/lexer_gen.go index e0da76fd..192bb43b 100644 --- a/internal/languages/lookahead/lexer_gen.go +++ b/internal/languages/lookahead/lexer_gen.go @@ -15,10 +15,7 @@ var Keyword_AsteriskAsterisk = core.NewTokenType( Keyword_AsteriskAsterisk_Idx, "**", "**", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "**") { return 2 @@ -34,10 +31,7 @@ var Keyword_Dot = core.NewTokenType( Keyword_Dot_Idx, ".", ".", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ".") { return 1 @@ -53,10 +47,7 @@ var Keyword_Slash = core.NewTokenType( Keyword_Slash_Idx, "/", "/", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "/") { return 1 @@ -72,10 +63,7 @@ var Keyword_Colon = core.NewTokenType( Keyword_Colon_Idx, ":", ":", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], ":") { return 1 @@ -91,10 +79,7 @@ var Keyword_ColonColon = core.NewTokenType( Keyword_ColonColon_Idx, "::", "::", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "::") { return 2 @@ -110,10 +95,7 @@ var Keyword_a = core.NewTokenType( Keyword_a_Idx, "a", "a", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "a") { return 1 @@ -129,10 +111,7 @@ var Keyword_b = core.NewTokenType( Keyword_b_Idx, "b", "b", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "b") { return 1 @@ -148,10 +127,7 @@ var Keyword_c = core.NewTokenType( Keyword_c_Idx, "c", "c", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "c") { return 1 @@ -167,10 +143,7 @@ var Keyword_d = core.NewTokenType( Keyword_d_Idx, "d", "d", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "d") { return 1 @@ -186,10 +159,7 @@ var Keyword_e = core.NewTokenType( Keyword_e_Idx, "e", "e", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "e") { return 1 @@ -205,10 +175,7 @@ var Keyword_f = core.NewTokenType( Keyword_f_Idx, "f", "f", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "f") { return 1 @@ -224,10 +191,7 @@ var Keyword_g = core.NewTokenType( Keyword_g_Idx, "g", "g", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "g") { return 1 @@ -243,10 +207,7 @@ var Keyword_h = core.NewTokenType( Keyword_h_Idx, "h", "h", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "h") { return 1 @@ -262,10 +223,7 @@ var Keyword_hello = core.NewTokenType( Keyword_hello_Idx, "hello", "hello", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "hello") { return 5 @@ -281,10 +239,7 @@ var Keyword_i = core.NewTokenType( Keyword_i_Idx, "i", "i", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "i") { return 1 @@ -300,10 +255,7 @@ var Keyword_world = core.NewTokenType( Keyword_world_Idx, "world", "world", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "world") { return 5 @@ -319,10 +271,7 @@ var Token_WS = core.NewTokenType( Token_WS_Idx, "WS", "WS", - core.SkippedGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -393,10 +342,7 @@ var Token_ID = core.NewTokenType( Token_ID_Idx, "ID", "ID", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -461,25 +407,31 @@ var Token_ID_Accepting = [2]bool{ 1: true, } +const ( + TokenMode_default = 0 +) + func NewLexer() lexer.Lexer { - return lexer.NewDefaultLexer( - Keyword_AsteriskAsterisk, - Keyword_Dot, - Keyword_Slash, - Keyword_Colon, - Keyword_ColonColon, - Keyword_a, - Keyword_b, - Keyword_c, - Keyword_d, - Keyword_e, - Keyword_f, - Keyword_g, - Keyword_h, - Keyword_hello, - Keyword_i, - Keyword_world, - Token_WS, - Token_ID, + modes := make([]*lexer.TokenMode, 1) + modes[TokenMode_default] = lexer.NewTokenMode("default", + lexer.UseTokenType(Keyword_AsteriskAsterisk), + lexer.UseTokenType(Keyword_Dot), + lexer.UseTokenType(Keyword_Slash), + lexer.UseTokenType(Keyword_Colon), + lexer.UseTokenType(Keyword_ColonColon), + lexer.UseTokenType(Keyword_a), + lexer.UseTokenType(Keyword_b), + lexer.UseTokenType(Keyword_c), + lexer.UseTokenType(Keyword_d), + lexer.UseTokenType(Keyword_e), + lexer.UseTokenType(Keyword_f), + lexer.UseTokenType(Keyword_g), + lexer.UseTokenType(Keyword_h), + lexer.UseTokenType(Keyword_hello), + lexer.UseTokenType(Keyword_i), + lexer.UseTokenType(Keyword_world), + lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier), + lexer.UseTokenType(Token_ID), ) + return lexer.NewDefaultLexer(TokenMode_default, modes...) } diff --git a/internal/languages/lookahead/lookahead.fb b/internal/languages/lookahead/lookahead.fb index e7c99a88..87849fcd 100644 --- a/internal/languages/lookahead/lookahead.fb +++ b/internal/languages/lookahead/lookahead.fb @@ -61,5 +61,5 @@ I returns Obj: "i" ( composite QualifiedNameRecursive: ID ("." QualifiedNameRecursive)?; -hidden token WS: /[ \t\r\n]+/; -token ID: /\w[\w0-9]*/; +hidden token WS: /[ \t\r\n]+/ +token ID: /\w[\w0-9]*/ \ No newline at end of file diff --git a/internal/languages/lookahead/parser_test.go b/internal/languages/lookahead/parser_test.go index 73e0a3cf..ac22cd0a 100644 --- a/internal/languages/lookahead/parser_test.go +++ b/internal/languages/lookahead/parser_test.go @@ -22,7 +22,7 @@ func BenchmarkNestedQualifiedNameLookahead(b *testing.B) { srv := CreateServices() lexerService := service.MustGet[lexer.Lexer](srv) parserService := service.MustGet[parser.Parser](srv) - tokens := lexerService.Lex(content).Tokens + tokens := lexerService.Exec(content).Tokens doc, err := fastbelt.NewDocumentFromString("file:///workspace/test.la", "lookahead", content) if err != nil { b.Fatal(err) diff --git a/internal/languages/token_groups/atn_gen.go b/internal/languages/token_groups/atn_gen.go index 1ea440c1..f1b4936c 100644 --- a/internal/languages/token_groups/atn_gen.go +++ b/internal/languages/token_groups/atn_gen.go @@ -257,7 +257,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[A__Basic_0], Keyword_a, nil), ) states[A__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[A__Basic_1], Token_Identifier, nil), + parser.NewAtomTransition(states[A__Basic_1], TokenGroup_Identifier, nil), ) states[A__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[A__Stop]), @@ -266,7 +266,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[B__Basic_3], Keyword_b, nil), ) states[B__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[B__Basic_1], Token_Identifier, nil), + parser.NewAtomTransition(states[B__Basic_1], TokenGroup_Identifier, nil), ) states[B__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[B__BlockEnd]), @@ -288,7 +288,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[C__Basic_0], Keyword_c, nil), ) states[C__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[C__Basic_1], Token_NestedIdentifier, nil), + parser.NewAtomTransition(states[C__Basic_1], TokenGroup_NestedIdentifier, nil), ) states[C__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[C__Stop]), @@ -297,7 +297,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[D__Basic_2], Keyword_d, nil), ) states[D__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[D__Basic_1], Token_Identifier, nil), + parser.NewAtomTransition(states[D__Basic_1], TokenGroup_Identifier, nil), ) states[D__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[D__Stop]), @@ -310,10 +310,10 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[E__Basic_0], Keyword_e, nil), ) states[E__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[E__Basic_1], Token_Identifier, nil), + parser.NewAtomTransition(states[E__Basic_1], TokenGroup_Identifier, nil), ) states[E__Basic_1].AppendTransitions( - parser.NewAtomTransition(states[E__Basic_2], Token_NestedIdentifier, nil), + parser.NewAtomTransition(states[E__Basic_2], TokenGroup_NestedIdentifier, nil), ) states[E__Basic_2].AppendTransitions( parser.NewEpsilonTransition(states[E__Stop]), @@ -322,7 +322,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[F__Basic_0], Keyword_f, nil), ) states[F__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[F__Basic_1], Token_KeywordGroup, nil), + parser.NewAtomTransition(states[F__Basic_1], TokenGroup_KeywordGroup, nil), ) states[F__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[F__Stop]), @@ -331,7 +331,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[G__Basic_0], Keyword_g, nil), ) states[G__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[G__Basic_1], Token_RegexGroup, nil), + parser.NewAtomTransition(states[G__Basic_1], TokenGroup_RegexGroup, nil), ) states[G__Basic_1].AppendTransitions( parser.NewEpsilonTransition(states[G__Stop]), @@ -340,7 +340,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewAtomTransition(states[H__Basic_4], Keyword_h, nil), ) states[H__Basic_0].AppendTransitions( - parser.NewAtomTransition(states[H_Value_a], Token_Identifier, nil), + parser.NewAtomTransition(states[H_Value_a], TokenGroup_Identifier, nil), ) states[H_Value_a].AppendTransitions( parser.NewAtomTransition(states[H__Basic_1], Keyword_a, nil), @@ -349,7 +349,7 @@ func BuildATN() *parser.RuntimeATN { parser.NewEpsilonTransition(states[H__BlockEnd]), ) states[H__Basic_2].AppendTransitions( - parser.NewAtomTransition(states[H_Value_b], Token_Identifier, nil), + parser.NewAtomTransition(states[H_Value_b], TokenGroup_Identifier, nil), ) states[H_Value_b].AppendTransitions( parser.NewAtomTransition(states[H__Basic_3], Keyword_b, nil), diff --git a/internal/languages/token_groups/completion_parser_gen.go b/internal/languages/token_groups/completion_parser_gen.go index 3e03d965..daf9c843 100644 --- a/internal/languages/token_groups/completion_parser_gen.go +++ b/internal/languages/token_groups/completion_parser_gen.go @@ -90,7 +90,7 @@ func (p *CompletionParser) ParseA() { } { p.cp.MarkAssignment("Value") - p.state.Consume(Token_Identifier) + p.state.Consume(TokenGroup_Identifier) p.cp.ClearAssignment() } } @@ -105,7 +105,7 @@ func (p *CompletionParser) ParseB() { p.cp.MarkAssignment("Value") switch prediction, _ := p.lookahead.BValueAlternatives(p.state); prediction { case 0: - p.state.Consume(Token_Identifier) + p.state.Consume(TokenGroup_Identifier) case 1: p.state.Consume(Keyword_b) } @@ -121,7 +121,7 @@ func (p *CompletionParser) ParseC() { } { p.cp.MarkAssignment("Value") - p.state.Consume(Token_NestedIdentifier) + p.state.Consume(TokenGroup_NestedIdentifier) p.cp.ClearAssignment() } } @@ -137,7 +137,7 @@ func (p *CompletionParser) ParseD() { p.state.Sync(D__Basic_2) if p.lookahead.DValueOptional(p.state) { p.cp.MarkAssignment("Value") - p.state.Consume(Token_Identifier) + p.state.Consume(TokenGroup_Identifier) p.cp.ClearAssignment() } } @@ -151,12 +151,12 @@ func (p *CompletionParser) ParseE() { } { p.cp.MarkAssignment("First") - p.state.Consume(Token_Identifier) + p.state.Consume(TokenGroup_Identifier) p.cp.ClearAssignment() } { p.cp.MarkAssignment("Second") - p.state.Consume(Token_NestedIdentifier) + p.state.Consume(TokenGroup_NestedIdentifier) p.cp.ClearAssignment() } } @@ -169,7 +169,7 @@ func (p *CompletionParser) ParseF() { } { p.cp.MarkAssignment("Value") - p.state.Consume(Token_KeywordGroup) + p.state.Consume(TokenGroup_KeywordGroup) p.cp.ClearAssignment() } } @@ -182,7 +182,7 @@ func (p *CompletionParser) ParseG() { } { p.cp.MarkAssignment("Value") - p.state.Consume(Token_RegexGroup) + p.state.Consume(TokenGroup_RegexGroup) p.cp.ClearAssignment() } } @@ -196,7 +196,7 @@ func (p *CompletionParser) ParseH() { switch prediction, failure := p.lookahead.HAlternatives(p.state); prediction { case 0: { - p.state.Consume(Token_Identifier) + p.state.Consume(TokenGroup_Identifier) } { p.cp.MarkAssignment("Value") @@ -205,7 +205,7 @@ func (p *CompletionParser) ParseH() { } case 1: { - p.state.Consume(Token_Identifier) + p.state.Consume(TokenGroup_Identifier) } { p.cp.MarkAssignment("Value") diff --git a/internal/languages/token_groups/lexer_gen.go b/internal/languages/token_groups/lexer_gen.go index 860c4b5e..47fddaeb 100644 --- a/internal/languages/token_groups/lexer_gen.go +++ b/internal/languages/token_groups/lexer_gen.go @@ -15,10 +15,7 @@ var Keyword_a = core.NewTokenType( Keyword_a_Idx, "a", "a", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "a") { return 1 @@ -34,10 +31,7 @@ var Keyword_b = core.NewTokenType( Keyword_b_Idx, "b", "b", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "b") { return 1 @@ -53,10 +47,7 @@ var Keyword_c = core.NewTokenType( Keyword_c_Idx, "c", "c", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "c") { return 1 @@ -72,10 +63,7 @@ var Keyword_d = core.NewTokenType( Keyword_d_Idx, "d", "d", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "d") { return 1 @@ -91,10 +79,7 @@ var Keyword_e = core.NewTokenType( Keyword_e_Idx, "e", "e", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "e") { return 1 @@ -110,10 +95,7 @@ var Keyword_f = core.NewTokenType( Keyword_f_Idx, "f", "f", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "f") { return 1 @@ -129,10 +111,7 @@ var Keyword_g = core.NewTokenType( Keyword_g_Idx, "g", "g", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "g") { return 1 @@ -148,10 +127,7 @@ var Keyword_h = core.NewTokenType( Keyword_h_Idx, "h", "h", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "h") { return 1 @@ -167,10 +143,7 @@ var Keyword_nested = core.NewTokenType( Keyword_nested_Idx, "nested", "nested", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "nested") { return 6 @@ -186,10 +159,7 @@ var Keyword_one = core.NewTokenType( Keyword_one_Idx, "one", "one", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "one") { return 3 @@ -205,10 +175,7 @@ var Keyword_two = core.NewTokenType( Keyword_two_Idx, "two", "two", - 0, core.TokenKindKeyword, - 0, - false, func(text string, offset int) int { if strings.HasPrefix(text[offset:], "two") { return 3 @@ -224,10 +191,7 @@ var Token_ID = core.NewTokenType( Token_ID_Idx, "ID", "ID", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -298,10 +262,7 @@ var Token_INT = core.NewTokenType( Token_INT_Idx, "INT", "INT", - 0, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -372,10 +333,7 @@ var Token_WS = core.NewTokenType( Token_WS_Idx, "WS", "WS", - core.SkippedGroup, core.TokenKindToken, - 0, - false, func(s string, offset int) int { input := s[offset:] length := len(input) @@ -440,10 +398,10 @@ var Token_WS_Accepting = [2]bool{ 1: true, } -const Token_Identifier_Idx = 15 +const TokenGroup_Identifier_Idx = 15 -var Token_Identifier = core.NewTokenGroup( - Token_Identifier_Idx, +var TokenGroup_Identifier = core.NewTokenGroup( + TokenGroup_Identifier_Idx, "Identifier", "Identifier", []*core.TokenType{ @@ -453,10 +411,10 @@ var Token_Identifier = core.NewTokenGroup( }, ) -const Token_KeywordGroup_Idx = 16 +const TokenGroup_KeywordGroup_Idx = 16 -var Token_KeywordGroup = core.NewTokenGroup( - Token_KeywordGroup_Idx, +var TokenGroup_KeywordGroup = core.NewTokenGroup( + TokenGroup_KeywordGroup_Idx, "KeywordGroup", "KeywordGroup", []*core.TokenType{ @@ -465,22 +423,22 @@ var Token_KeywordGroup = core.NewTokenGroup( }, ) -const Token_NestedIdentifier_Idx = 17 +const TokenGroup_NestedIdentifier_Idx = 17 -var Token_NestedIdentifier = core.NewTokenGroup( - Token_NestedIdentifier_Idx, +var TokenGroup_NestedIdentifier = core.NewTokenGroup( + TokenGroup_NestedIdentifier_Idx, "NestedIdentifier", "NestedIdentifier", []*core.TokenType{ Keyword_nested, - Token_Identifier, + TokenGroup_Identifier, }, ) -const Token_RegexGroup_Idx = 18 +const TokenGroup_RegexGroup_Idx = 18 -var Token_RegexGroup = core.NewTokenGroup( - Token_RegexGroup_Idx, +var TokenGroup_RegexGroup = core.NewTokenGroup( + TokenGroup_RegexGroup_Idx, "RegexGroup", "RegexGroup", []*core.TokenType{ @@ -489,21 +447,27 @@ var Token_RegexGroup = core.NewTokenGroup( }, ) +const ( + TokenMode_default = 0 +) + func NewLexer() lexer.Lexer { - return lexer.NewDefaultLexer( - Keyword_a, - Keyword_b, - Keyword_c, - Keyword_d, - Keyword_e, - Keyword_f, - Keyword_g, - Keyword_h, - Keyword_nested, - Keyword_one, - Keyword_two, - Token_ID, - Token_INT, - Token_WS, + modes := make([]*lexer.TokenMode, 1) + modes[TokenMode_default] = lexer.NewTokenMode("default", + lexer.UseTokenType(Keyword_a), + lexer.UseTokenType(Keyword_nested), + lexer.UseTokenType(Keyword_one), + lexer.UseTokenType(Keyword_two), + lexer.UseTokenType(Keyword_b), + lexer.UseTokenType(Keyword_c), + lexer.UseTokenType(Keyword_d), + lexer.UseTokenType(Keyword_e), + lexer.UseTokenType(Keyword_f), + lexer.UseTokenType(Keyword_g), + lexer.UseTokenType(Keyword_h), + lexer.UseTokenType(Token_ID), + lexer.UseTokenType(Token_INT), + lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier), ) + return lexer.NewDefaultLexer(TokenMode_default, modes...) } diff --git a/internal/languages/token_groups/parser_gen.go b/internal/languages/token_groups/parser_gen.go index 35aeaaa5..f8a5c539 100644 --- a/internal/languages/token_groups/parser_gen.go +++ b/internal/languages/token_groups/parser_gen.go @@ -110,7 +110,7 @@ func (p *Parser) ParseA() Item { core.AssignToken(current, token, A_a) } { - token := p.state.Consume(Token_Identifier) + token := p.state.Consume(TokenGroup_Identifier) core.AssignToken(current, token, A__Basic_0) if token != nil { current.SetValue(token) @@ -132,7 +132,7 @@ func (p *Parser) ParseB() Item { { switch prediction, _ := p.lookahead.BValueAlternatives(p.state); prediction { case 0: - token := p.state.Consume(Token_Identifier) + token := p.state.Consume(TokenGroup_Identifier) core.AssignToken(current, token, B__Basic_0) if token != nil { current.SetValue(token) @@ -159,7 +159,7 @@ func (p *Parser) ParseC() Item { core.AssignToken(current, token, C_c) } { - token := p.state.Consume(Token_NestedIdentifier) + token := p.state.Consume(TokenGroup_NestedIdentifier) core.AssignToken(current, token, C__Basic_0) if token != nil { current.SetValue(token) @@ -181,7 +181,7 @@ func (p *Parser) ParseD() Item { { p.state.Sync(D__Basic_2) if p.lookahead.DValueOptional(p.state) { - token := p.state.Consume(Token_Identifier) + token := p.state.Consume(TokenGroup_Identifier) core.AssignToken(current, token, D__Basic_0) if token != nil { current.SetValue(token) @@ -202,14 +202,14 @@ func (p *Parser) ParseE() Recovery { core.AssignToken(current, token, E_e) } { - token := p.state.Consume(Token_Identifier) + token := p.state.Consume(TokenGroup_Identifier) core.AssignToken(current, token, E__Basic_0) if token != nil { current.SetFirst(token) } } { - token := p.state.Consume(Token_NestedIdentifier) + token := p.state.Consume(TokenGroup_NestedIdentifier) core.AssignToken(current, token, E__Basic_1) if token != nil { current.SetSecond(token) @@ -229,7 +229,7 @@ func (p *Parser) ParseF() Item { core.AssignToken(current, token, F_f) } { - token := p.state.Consume(Token_KeywordGroup) + token := p.state.Consume(TokenGroup_KeywordGroup) core.AssignToken(current, token, F__Basic_0) if token != nil { current.SetValue(token) @@ -249,7 +249,7 @@ func (p *Parser) ParseG() Item { core.AssignToken(current, token, G_g) } { - token := p.state.Consume(Token_RegexGroup) + token := p.state.Consume(TokenGroup_RegexGroup) core.AssignToken(current, token, G__Basic_0) if token != nil { current.SetValue(token) @@ -271,7 +271,7 @@ func (p *Parser) ParseH() Item { switch prediction, failure := p.lookahead.HAlternatives(p.state); prediction { case 0: { - token := p.state.Consume(Token_Identifier) + token := p.state.Consume(TokenGroup_Identifier) core.AssignToken(current, token, H__Basic_0) } { @@ -283,7 +283,7 @@ func (p *Parser) ParseH() Item { } case 1: { - token := p.state.Consume(Token_Identifier) + token := p.state.Consume(TokenGroup_Identifier) core.AssignToken(current, token, H__Basic_2) } { diff --git a/internal/languages/token_groups/parser_lookahead_gen.go b/internal/languages/token_groups/parser_lookahead_gen.go index 66364d5b..964264d4 100644 --- a/internal/languages/token_groups/parser_lookahead_gen.go +++ b/internal/languages/token_groups/parser_lookahead_gen.go @@ -12,7 +12,7 @@ const ( ) var BValueAlternatives = parser.LL1Lookahead{ - Types: []*core.TokenType{Token_Identifier, Keyword_b}, + Types: []*core.TokenType{TokenGroup_Identifier, Keyword_b}, Lookup: []int{1: 1, 2: 2, 12: 1, 13: 1}, } @@ -49,7 +49,7 @@ func (l *DefaultTokenGroupsParserLookahead) BValueAlternatives(state *parser.Par } func (l *DefaultTokenGroupsParserLookahead) DValueOptional(state *parser.ParserState) bool { - return Token_Identifier.Matches(state.LA(1).Type) + return TokenGroup_Identifier.Matches(state.LA(1).Type) } func (l *DefaultTokenGroupsParserLookahead) HAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { diff --git a/internal/languages/token_groups/token_groups.fb b/internal/languages/token_groups/token_groups.fb index 7fe3d29a..6ebb2a75 100644 --- a/internal/languages/token_groups/token_groups.fb +++ b/internal/languages/token_groups/token_groups.fb @@ -58,4 +58,4 @@ token group RegexGroup { token ID: /[a-zA-Z_][a-zA-Z0-9_]*/; token INT: /[0-9]+/; -hidden token WS: /\s+/; +hidden token WS: /\s+/; \ No newline at end of file diff --git a/internal/languages/token_modes/atn_gen.go b/internal/languages/token_modes/atn_gen.go new file mode 100644 index 00000000..0ec4168c --- /dev/null +++ b/internal/languages/token_modes/atn_gen.go @@ -0,0 +1,425 @@ +// Code generated by fastbelt. DO NOT EDIT. + +package token_modes + +import ( + "sync" + "typefox.dev/fastbelt/parser" +) + +const ( + Model__Start = iota + Model__Stop + Statement__Start + Statement__Stop + VariableDecl__Start + VariableDecl__Stop + Expression__Start + Expression__Stop + Additive__Start + Additive__Stop + Primary__Start + Primary__Stop + Parentheses__Start + Parentheses__Stop + VariableRef__Start + VariableRef__Stop + NumericLiteral__Start + NumericLiteral__Stop + StringLiteral__Start + StringLiteral__Stop + StringContent__Start + StringContent__Stop + StringText__Start + StringText__Stop + Interpolation__Start + Interpolation__Stop + Model__Basic_0 + Model__Basic_1 + Model__LoopEntry + Model__LoopEnd + Model__LoopBack + Statement__Basic_0 + Statement__Basic_1 + VariableDecl_Name_ID + VariableDecl_ColonEquals + VariableDecl__Basic_0 + VariableDecl__Basic_1 + Expression__Basic_0 + Expression__Basic_1 + Additive__Basic_0 + Additive_Operator_Plus + Additive__Basic_1 + Additive__Basic_2 + Additive__LoopEntry + Additive__LoopEnd + Additive__LoopBack + Primary__Basic_0 + Primary__Basic_1 + Primary__Basic_2 + Primary__Basic_3 + Primary__Basic_4 + Primary__Basic_5 + Primary__Basic_6 + Primary__Basic_7 + Primary__Basic_8 + Primary__BlockEnd + Parentheses_LeftParen + Parentheses__Basic_0 + Parentheses_RightParen + Parentheses__Basic_1 + VariableRef_Name_ID + VariableRef__Basic + NumericLiteral_Value_INT + NumericLiteral__Basic + StringLiteral_Backtick_0 + StringLiteral__Basic_0 + StringLiteral__Basic_1 + StringLiteral__LoopEntry + StringLiteral__LoopEnd + StringLiteral__LoopBack + StringLiteral_Backtick_1 + StringLiteral__Basic_2 + StringContent__Basic_0 + StringContent__Basic_1 + StringContent__Basic_2 + StringContent__Basic_3 + StringContent__Basic_4 + StringContent__BlockEnd + StringText_Value_STRING_CONTENT + StringText__Basic + Interpolation_HashLeftBrace + Interpolation__Basic_0 + Interpolation_RightBrace + Interpolation__Basic_1 +) + +var once sync.Once +var atn *parser.RuntimeATN + +func ATN() *parser.RuntimeATN { + once.Do(func() { + atn = BuildATN() + }) + return atn +} +func BuildATN() *parser.RuntimeATN { + states := make([]*parser.RuntimeATNState, 84) + states[Model__Start] = parser.NewATNState(Model__Start, parser.ATNRuleStart, true) + states[Model__Stop] = parser.NewATNState(Model__Stop, parser.ATNRuleStop, false) + states[Statement__Start] = parser.NewATNState(Statement__Start, parser.ATNRuleStart, true) + states[Statement__Stop] = parser.NewATNState(Statement__Stop, parser.ATNRuleStop, false) + states[VariableDecl__Start] = parser.NewATNState(VariableDecl__Start, parser.ATNRuleStart, true) + states[VariableDecl__Stop] = parser.NewATNState(VariableDecl__Stop, parser.ATNRuleStop, false) + states[Expression__Start] = parser.NewATNState(Expression__Start, parser.ATNRuleStart, true) + states[Expression__Stop] = parser.NewATNState(Expression__Stop, parser.ATNRuleStop, false) + states[Additive__Start] = parser.NewATNState(Additive__Start, parser.ATNRuleStart, true) + states[Additive__Stop] = parser.NewATNState(Additive__Stop, parser.ATNRuleStop, false) + states[Primary__Start] = parser.NewATNState(Primary__Start, parser.ATNRuleStart, true) + states[Primary__Stop] = parser.NewATNState(Primary__Stop, parser.ATNRuleStop, false) + states[Parentheses__Start] = parser.NewATNState(Parentheses__Start, parser.ATNRuleStart, true) + states[Parentheses__Stop] = parser.NewATNState(Parentheses__Stop, parser.ATNRuleStop, false) + states[VariableRef__Start] = parser.NewATNState(VariableRef__Start, parser.ATNRuleStart, true) + states[VariableRef__Stop] = parser.NewATNState(VariableRef__Stop, parser.ATNRuleStop, false) + states[NumericLiteral__Start] = parser.NewATNState(NumericLiteral__Start, parser.ATNRuleStart, true) + states[NumericLiteral__Stop] = parser.NewATNState(NumericLiteral__Stop, parser.ATNRuleStop, false) + states[StringLiteral__Start] = parser.NewATNState(StringLiteral__Start, parser.ATNRuleStart, true) + states[StringLiteral__Stop] = parser.NewATNState(StringLiteral__Stop, parser.ATNRuleStop, false) + states[StringContent__Start] = parser.NewATNState(StringContent__Start, parser.ATNRuleStart, true) + states[StringContent__Stop] = parser.NewATNState(StringContent__Stop, parser.ATNRuleStop, false) + states[StringText__Start] = parser.NewATNState(StringText__Start, parser.ATNRuleStart, true) + states[StringText__Stop] = parser.NewATNState(StringText__Stop, parser.ATNRuleStop, false) + states[Interpolation__Start] = parser.NewATNState(Interpolation__Start, parser.ATNRuleStart, true) + states[Interpolation__Stop] = parser.NewATNState(Interpolation__Stop, parser.ATNRuleStop, false) + states[Model__Basic_0] = parser.NewATNState(Model__Basic_0, parser.ATNBasic, true) + states[Model__Basic_1] = parser.NewATNState(Model__Basic_1, parser.ATNBasic, true) + states[Model__LoopEntry] = parser.NewATNState(Model__LoopEntry, parser.ATNLoopEntry, true).SetDecision(0) + states[Model__LoopEnd] = parser.NewATNState(Model__LoopEnd, parser.ATNLoopEnd, true) + states[Model__LoopBack] = parser.NewATNState(Model__LoopBack, parser.ATNLoopBack, true) + states[Statement__Basic_0] = parser.NewATNState(Statement__Basic_0, parser.ATNBasic, true) + states[Statement__Basic_1] = parser.NewATNState(Statement__Basic_1, parser.ATNBasic, true) + states[VariableDecl_Name_ID] = parser.NewATNState(VariableDecl_Name_ID, parser.ATNBasic, false) + states[VariableDecl_ColonEquals] = parser.NewATNState(VariableDecl_ColonEquals, parser.ATNBasic, false) + states[VariableDecl__Basic_0] = parser.NewATNState(VariableDecl__Basic_0, parser.ATNBasic, true) + states[VariableDecl__Basic_1] = parser.NewATNState(VariableDecl__Basic_1, parser.ATNBasic, true) + states[Expression__Basic_0] = parser.NewATNState(Expression__Basic_0, parser.ATNBasic, true) + states[Expression__Basic_1] = parser.NewATNState(Expression__Basic_1, parser.ATNBasic, true) + states[Additive__Basic_0] = parser.NewATNState(Additive__Basic_0, parser.ATNBasic, true) + states[Additive_Operator_Plus] = parser.NewATNState(Additive_Operator_Plus, parser.ATNBasic, false) + states[Additive__Basic_1] = parser.NewATNState(Additive__Basic_1, parser.ATNBasic, true) + states[Additive__Basic_2] = parser.NewATNState(Additive__Basic_2, parser.ATNBasic, true) + states[Additive__LoopEntry] = parser.NewATNState(Additive__LoopEntry, parser.ATNLoopEntry, true).SetDecision(1) + states[Additive__LoopEnd] = parser.NewATNState(Additive__LoopEnd, parser.ATNLoopEnd, true) + states[Additive__LoopBack] = parser.NewATNState(Additive__LoopBack, parser.ATNLoopBack, true) + states[Primary__Basic_0] = parser.NewATNState(Primary__Basic_0, parser.ATNBasic, true) + states[Primary__Basic_1] = parser.NewATNState(Primary__Basic_1, parser.ATNBasic, true) + states[Primary__Basic_2] = parser.NewATNState(Primary__Basic_2, parser.ATNBasic, true) + states[Primary__Basic_3] = parser.NewATNState(Primary__Basic_3, parser.ATNBasic, true) + states[Primary__Basic_4] = parser.NewATNState(Primary__Basic_4, parser.ATNBasic, true) + states[Primary__Basic_5] = parser.NewATNState(Primary__Basic_5, parser.ATNBasic, true) + states[Primary__Basic_6] = parser.NewATNState(Primary__Basic_6, parser.ATNBasic, true) + states[Primary__Basic_7] = parser.NewATNState(Primary__Basic_7, parser.ATNBasic, true) + states[Primary__Basic_8] = parser.NewATNState(Primary__Basic_8, parser.ATNBasic, true).SetDecision(2) + states[Primary__BlockEnd] = parser.NewATNState(Primary__BlockEnd, parser.ATNBlockEnd, true) + states[Parentheses_LeftParen] = parser.NewATNState(Parentheses_LeftParen, parser.ATNBasic, false) + states[Parentheses__Basic_0] = parser.NewATNState(Parentheses__Basic_0, parser.ATNBasic, true) + states[Parentheses_RightParen] = parser.NewATNState(Parentheses_RightParen, parser.ATNBasic, false) + states[Parentheses__Basic_1] = parser.NewATNState(Parentheses__Basic_1, parser.ATNBasic, true) + states[VariableRef_Name_ID] = parser.NewATNState(VariableRef_Name_ID, parser.ATNBasic, false) + states[VariableRef__Basic] = parser.NewATNState(VariableRef__Basic, parser.ATNBasic, true) + states[NumericLiteral_Value_INT] = parser.NewATNState(NumericLiteral_Value_INT, parser.ATNBasic, false) + states[NumericLiteral__Basic] = parser.NewATNState(NumericLiteral__Basic, parser.ATNBasic, true) + states[StringLiteral_Backtick_0] = parser.NewATNState(StringLiteral_Backtick_0, parser.ATNBasic, false) + states[StringLiteral__Basic_0] = parser.NewATNState(StringLiteral__Basic_0, parser.ATNBasic, true) + states[StringLiteral__Basic_1] = parser.NewATNState(StringLiteral__Basic_1, parser.ATNBasic, true) + states[StringLiteral__LoopEntry] = parser.NewATNState(StringLiteral__LoopEntry, parser.ATNLoopEntry, true).SetDecision(3) + states[StringLiteral__LoopEnd] = parser.NewATNState(StringLiteral__LoopEnd, parser.ATNLoopEnd, true) + states[StringLiteral__LoopBack] = parser.NewATNState(StringLiteral__LoopBack, parser.ATNLoopBack, true) + states[StringLiteral_Backtick_1] = parser.NewATNState(StringLiteral_Backtick_1, parser.ATNBasic, false) + states[StringLiteral__Basic_2] = parser.NewATNState(StringLiteral__Basic_2, parser.ATNBasic, true) + states[StringContent__Basic_0] = parser.NewATNState(StringContent__Basic_0, parser.ATNBasic, true) + states[StringContent__Basic_1] = parser.NewATNState(StringContent__Basic_1, parser.ATNBasic, true) + states[StringContent__Basic_2] = parser.NewATNState(StringContent__Basic_2, parser.ATNBasic, true) + states[StringContent__Basic_3] = parser.NewATNState(StringContent__Basic_3, parser.ATNBasic, true) + states[StringContent__Basic_4] = parser.NewATNState(StringContent__Basic_4, parser.ATNBasic, true).SetDecision(4) + states[StringContent__BlockEnd] = parser.NewATNState(StringContent__BlockEnd, parser.ATNBlockEnd, true) + states[StringText_Value_STRING_CONTENT] = parser.NewATNState(StringText_Value_STRING_CONTENT, parser.ATNBasic, false) + states[StringText__Basic] = parser.NewATNState(StringText__Basic, parser.ATNBasic, true) + states[Interpolation_HashLeftBrace] = parser.NewATNState(Interpolation_HashLeftBrace, parser.ATNBasic, false) + states[Interpolation__Basic_0] = parser.NewATNState(Interpolation__Basic_0, parser.ATNBasic, true) + states[Interpolation_RightBrace] = parser.NewATNState(Interpolation_RightBrace, parser.ATNBasic, false) + states[Interpolation__Basic_1] = parser.NewATNState(Interpolation__Basic_1, parser.ATNBasic, true) + states[Model__Start].AppendTransitions( + parser.NewEpsilonTransition(states[Model__LoopEntry]), + ) + states[Statement__Start].AppendTransitions( + parser.NewEpsilonTransition(states[Statement__Basic_0]), + ) + states[VariableDecl__Start].AppendTransitions( + parser.NewEpsilonTransition(states[VariableDecl_Name_ID]), + ) + states[Expression__Start].AppendTransitions( + parser.NewEpsilonTransition(states[Expression__Basic_0]), + ) + states[Additive__Start].AppendTransitions( + parser.NewEpsilonTransition(states[Additive__Basic_0]), + ) + states[Primary__Start].AppendTransitions( + parser.NewEpsilonTransition(states[Primary__Basic_8]), + ) + states[Parentheses__Start].AppendTransitions( + parser.NewEpsilonTransition(states[Parentheses_LeftParen]), + ) + states[VariableRef__Start].AppendTransitions( + parser.NewEpsilonTransition(states[VariableRef_Name_ID]), + ) + states[NumericLiteral__Start].AppendTransitions( + parser.NewEpsilonTransition(states[NumericLiteral_Value_INT]), + ) + states[StringLiteral__Start].AppendTransitions( + parser.NewEpsilonTransition(states[StringLiteral_Backtick_0]), + ) + states[StringContent__Start].AppendTransitions( + parser.NewEpsilonTransition(states[StringContent__Basic_4]), + ) + states[StringText__Start].AppendTransitions( + parser.NewEpsilonTransition(states[StringText_Value_STRING_CONTENT]), + ) + states[Interpolation__Start].AppendTransitions( + parser.NewEpsilonTransition(states[Interpolation_HashLeftBrace]), + ) + states[Model__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[Statement__Start], states[Model__Basic_1], nil), + ) + states[Model__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[Model__LoopBack]), + ) + states[Model__LoopEntry].AppendTransitions( + parser.NewEpsilonTransition(states[Model__Basic_0]), + parser.NewEpsilonTransition(states[Model__LoopEnd]), + ) + states[Model__LoopEnd].AppendTransitions( + parser.NewEpsilonTransition(states[Model__Stop]), + ) + states[Model__LoopBack].AppendTransitions( + parser.NewEpsilonTransition(states[Model__LoopEntry]), + ) + states[Statement__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[VariableDecl__Start], states[Statement__Basic_1], nil), + ) + states[Statement__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[Statement__Stop]), + ) + states[VariableDecl_Name_ID].AppendTransitions( + parser.NewAtomTransition(states[VariableDecl_ColonEquals], Token_ID, nil), + ) + states[VariableDecl_ColonEquals].AppendTransitions( + parser.NewAtomTransition(states[VariableDecl__Basic_0], Keyword_ColonEquals, nil), + ) + states[VariableDecl__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[Expression__Start], states[VariableDecl__Basic_1], nil), + ) + states[VariableDecl__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[VariableDecl__Stop]), + ) + states[Expression__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[Additive__Start], states[Expression__Basic_1], nil), + ) + states[Expression__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[Expression__Stop]), + ) + states[Additive__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[Primary__Start], states[Additive__LoopEntry], nil), + ) + states[Additive_Operator_Plus].AppendTransitions( + parser.NewAtomTransition(states[Additive__Basic_1], Keyword_Plus, nil), + ) + states[Additive__Basic_1].AppendTransitions( + parser.NewRuleTransition(states[Primary__Start], states[Additive__Basic_2], nil), + ) + states[Additive__Basic_2].AppendTransitions( + parser.NewEpsilonTransition(states[Additive__LoopBack]), + ) + states[Additive__LoopEntry].AppendTransitions( + parser.NewEpsilonTransition(states[Additive_Operator_Plus]), + parser.NewEpsilonTransition(states[Additive__LoopEnd]), + ) + states[Additive__LoopEnd].AppendTransitions( + parser.NewEpsilonTransition(states[Additive__Stop]), + ) + states[Additive__LoopBack].AppendTransitions( + parser.NewEpsilonTransition(states[Additive__LoopEntry]), + ) + states[Primary__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[VariableRef__Start], states[Primary__Basic_1], nil), + ) + states[Primary__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[Primary__BlockEnd]), + ) + states[Primary__Basic_2].AppendTransitions( + parser.NewRuleTransition(states[NumericLiteral__Start], states[Primary__Basic_3], nil), + ) + states[Primary__Basic_3].AppendTransitions( + parser.NewEpsilonTransition(states[Primary__BlockEnd]), + ) + states[Primary__Basic_4].AppendTransitions( + parser.NewRuleTransition(states[StringLiteral__Start], states[Primary__Basic_5], nil), + ) + states[Primary__Basic_5].AppendTransitions( + parser.NewEpsilonTransition(states[Primary__BlockEnd]), + ) + states[Primary__Basic_6].AppendTransitions( + parser.NewRuleTransition(states[Parentheses__Start], states[Primary__Basic_7], nil), + ) + states[Primary__Basic_7].AppendTransitions( + parser.NewEpsilonTransition(states[Primary__BlockEnd]), + ) + states[Primary__Basic_8].AppendTransitions( + parser.NewEpsilonTransition(states[Primary__Basic_0]), + parser.NewEpsilonTransition(states[Primary__Basic_2]), + parser.NewEpsilonTransition(states[Primary__Basic_4]), + parser.NewEpsilonTransition(states[Primary__Basic_6]), + ) + states[Primary__BlockEnd].AppendTransitions( + parser.NewEpsilonTransition(states[Primary__Stop]), + ) + states[Parentheses_LeftParen].AppendTransitions( + parser.NewAtomTransition(states[Parentheses__Basic_0], Keyword_LeftParen, nil), + ) + states[Parentheses__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[Expression__Start], states[Parentheses_RightParen], nil), + ) + states[Parentheses_RightParen].AppendTransitions( + parser.NewAtomTransition(states[Parentheses__Basic_1], Keyword_RightParen, nil), + ) + states[Parentheses__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[Parentheses__Stop]), + ) + states[VariableRef_Name_ID].AppendTransitions( + parser.NewAtomTransition(states[VariableRef__Basic], Token_ID, &parser.CompletionHint{Field: "VariableRef.Name"}), + ) + states[VariableRef__Basic].AppendTransitions( + parser.NewEpsilonTransition(states[VariableRef__Stop]), + ) + states[NumericLiteral_Value_INT].AppendTransitions( + parser.NewAtomTransition(states[NumericLiteral__Basic], Token_INT, nil), + ) + states[NumericLiteral__Basic].AppendTransitions( + parser.NewEpsilonTransition(states[NumericLiteral__Stop]), + ) + states[StringLiteral_Backtick_0].AppendTransitions( + parser.NewAtomTransition(states[StringLiteral__LoopEntry], Keyword_Backtick, nil), + ) + states[StringLiteral__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[StringContent__Start], states[StringLiteral__Basic_1], nil), + ) + states[StringLiteral__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[StringLiteral__LoopBack]), + ) + states[StringLiteral__LoopEntry].AppendTransitions( + parser.NewEpsilonTransition(states[StringLiteral__Basic_0]), + parser.NewEpsilonTransition(states[StringLiteral__LoopEnd]), + ) + states[StringLiteral__LoopEnd].AppendTransitions( + parser.NewEpsilonTransition(states[StringLiteral_Backtick_1]), + ) + states[StringLiteral__LoopBack].AppendTransitions( + parser.NewEpsilonTransition(states[StringLiteral__LoopEntry]), + ) + states[StringLiteral_Backtick_1].AppendTransitions( + parser.NewAtomTransition(states[StringLiteral__Basic_2], Keyword_Backtick, nil), + ) + states[StringLiteral__Basic_2].AppendTransitions( + parser.NewEpsilonTransition(states[StringLiteral__Stop]), + ) + states[StringContent__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[StringText__Start], states[StringContent__Basic_1], nil), + ) + states[StringContent__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[StringContent__BlockEnd]), + ) + states[StringContent__Basic_2].AppendTransitions( + parser.NewRuleTransition(states[Interpolation__Start], states[StringContent__Basic_3], nil), + ) + states[StringContent__Basic_3].AppendTransitions( + parser.NewEpsilonTransition(states[StringContent__BlockEnd]), + ) + states[StringContent__Basic_4].AppendTransitions( + parser.NewEpsilonTransition(states[StringContent__Basic_0]), + parser.NewEpsilonTransition(states[StringContent__Basic_2]), + ) + states[StringContent__BlockEnd].AppendTransitions( + parser.NewEpsilonTransition(states[StringContent__Stop]), + ) + states[StringText_Value_STRING_CONTENT].AppendTransitions( + parser.NewAtomTransition(states[StringText__Basic], Token_STRING_CONTENT, nil), + ) + states[StringText__Basic].AppendTransitions( + parser.NewEpsilonTransition(states[StringText__Stop]), + ) + states[Interpolation_HashLeftBrace].AppendTransitions( + parser.NewAtomTransition(states[Interpolation__Basic_0], Keyword_HashLeftBrace, nil), + ) + states[Interpolation__Basic_0].AppendTransitions( + parser.NewRuleTransition(states[Expression__Start], states[Interpolation_RightBrace], nil), + ) + states[Interpolation_RightBrace].AppendTransitions( + parser.NewAtomTransition(states[Interpolation__Basic_1], Keyword_RightBrace, nil), + ) + states[Interpolation__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[Interpolation__Stop]), + ) + decisionStates := make([]*parser.RuntimeATNState, 5) + decisionStates[0] = states[Model__LoopEntry] + decisionStates[1] = states[Additive__LoopEntry] + decisionStates[2] = states[Primary__Basic_8] + decisionStates[3] = states[StringLiteral__LoopEntry] + decisionStates[4] = states[StringContent__Basic_4] + decisionMap := make([]*parser.RuntimeATNState, 5) + decisionMap[0] = states[Model__LoopEntry] + decisionMap[1] = states[Additive__LoopEntry] + decisionMap[2] = states[Primary__Basic_8] + decisionMap[3] = states[StringLiteral__LoopEntry] + decisionMap[4] = states[StringContent__Basic_4] + return parser.NewRuntimeATN(states, decisionStates, decisionMap) +} diff --git a/internal/languages/token_modes/benchmark_test.go b/internal/languages/token_modes/benchmark_test.go new file mode 100644 index 00000000..b31aa4f3 --- /dev/null +++ b/internal/languages/token_modes/benchmark_test.go @@ -0,0 +1,45 @@ +package token_modes + +import ( + "strings" + "testing" + + "typefox.dev/fastbelt" + "typefox.dev/fastbelt/lexer" + "typefox.dev/fastbelt/util/service" +) + +func BenchmarkNestedString(b *testing.B) { + content, _ := generateNestedString() + srv := CreateServices() + lexerService := service.MustGet[lexer.Lexer](srv) + doc, err := fastbelt.NewDocumentFromString("file:///workspace/nested_string.mode", "modes", content) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(content))) + b.ResetTimer() + for b.Loop() { + lexerResult := lexerService.Exec(content) + doc.Tokens = lexerResult.Tokens + } +} + +func generateNestedString() (string, error) { + content := strings.Builder{} + + var recursion func(depth int) + recursion = func(depth int) { + if depth == 0 { + content.WriteString("`EXIT!!!`") + return + } + content.WriteString("`Hello, World! #{") + recursion(depth - 1) + content.WriteString("} How are you?`") + } + + recursion(10000) + + return "VAR := " + content.String(), nil +} diff --git a/internal/languages/token_modes/completion_gen.go b/internal/languages/token_modes/completion_gen.go new file mode 100644 index 00000000..11921a34 --- /dev/null +++ b/internal/languages/token_modes/completion_gen.go @@ -0,0 +1,107 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes + +import ( + "context" + "iter" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/parser" + "typefox.dev/fastbelt/util/service" +) + +// TokenModesCompletionFilter lets adopters refine the candidates produced by +// the existing TokenModesScopeProvider when completing a cross-reference. +type TokenModesCompletionFilter interface { + FilterVariableRefName(ctx context.Context, reference *core.Reference[VariableDecl], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] +} + +type DefaultTokenModesCompletionFilter struct{} + +func NewDefaultTokenModesCompletionFilter() TokenModesCompletionFilter { + return &DefaultTokenModesCompletionFilter{} +} + +func (*DefaultTokenModesCompletionFilter) FilterVariableRefName(_ context.Context, _ *core.Reference[VariableDecl], in iter.Seq[*core.SymbolDescription]) iter.Seq[*core.SymbolDescription] { + return in +} + +type TokenModesCompletionDispatchFunc func( + ctx context.Context, + sc *service.Container, + owner core.AstNode, +) iter.Seq[*core.SymbolDescription] + +var TokenModesCompletionDispatch = map[string]TokenModesCompletionDispatchFunc{ + "VariableRef.Name": func(ctx context.Context, sc *service.Container, owner core.AstNode) iter.Seq[*core.SymbolDescription] { + typedOwner, ok := owner.(VariableRef) + if !ok { + return func(yield func(*core.SymbolDescription) bool) {} + } + refs := service.MustGet[TokenModesReferencesConstructor](sc) + scopes := service.MustGet[TokenModesScopeProvider](sc) + filter := service.MustGet[TokenModesCompletionFilter](sc) + ref := refs.VariableRefName(typedOwner, nil) + candidates := scopes.ScopeVariableRefName(ctx, ref).AllElements() + return filter.FilterVariableRefName(ctx, ref, candidates) + }, +} + +type TokenModesCompletionAdapter struct { + sc *service.Container +} + +func NewTokenModesCompletionAdapter(sc *service.Container) *TokenModesCompletionAdapter { + return &TokenModesCompletionAdapter{sc: sc} +} + +func (a *TokenModesCompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult { + return NewCompletionParser(a.sc).Parse(tokens) +} + +func (a *TokenModesCompletionAdapter) ATN() *parser.RuntimeATN { + return BuildATN() +} + +func (a *TokenModesCompletionAdapter) SyntheticOwnerFor(ruleKey string) (core.AstNode, bool) { + factory, ok := TokenModesSyntheticFactories[ruleKey] + if !ok { + return nil, false + } + return factory(), true +} + +func (a *TokenModesCompletionAdapter) DispatchCompletion(ctx context.Context, field string, owner core.AstNode) (iter.Seq[*core.SymbolDescription], bool) { + dispatch, ok := TokenModesCompletionDispatch[field] + if !ok { + return nil, false + } + return dispatch(ctx, a.sc, owner), true +} + +func (a *TokenModesCompletionAdapter) HasAssignment(node core.AstNode, property string) bool { + switch n := node.(type) { + case VariableRef: + switch property { + case "Name": + return n.Name() != nil + } + } + return false +} + +func (a *TokenModesCompletionAdapter) ApplyAction(actionType, property string, value core.AstNode) core.AstNode { + switch actionType { + case "BinaryExpression": + node := NewBinaryExpression() + switch property { + case "Left": + if v, ok := value.(Expression); ok { + node.SetLeft(v) + } + } + return node + } + return nil +} diff --git a/internal/languages/token_modes/completion_parser_gen.go b/internal/languages/token_modes/completion_parser_gen.go new file mode 100644 index 00000000..2da8936c --- /dev/null +++ b/internal/languages/token_modes/completion_parser_gen.go @@ -0,0 +1,270 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes + +import ( + "sync" + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/parser" + "typefox.dev/fastbelt/util/service" +) + +type CompletionParser struct { + state *parser.ParserState + cp *parser.CompletionParserState + sc *service.Container + atn func() *parser.RuntimeATN + lookahead TokenModesParserLookahead +} + +func NewCompletionParser(sc *service.Container) *CompletionParser { + return &CompletionParser{ + sc: sc, + atn: sync.OnceValue(BuildATN), + } +} + +// Parse runs the completion parser over the given prefix tokens (typically +// the document's tokens up to the cursor) and returns the recorded +// snapshots and rule stack. The completion provider feeds that result into +// the ATN simulator. +func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult { + messages := service.MustGet[parser.ErrorMessageProvider](p.sc) + recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) + lookahead := service.MustGet[TokenModesParserLookahead](p.sc) + cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead} + cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages) + cp.cp = parser.NewCompletionParserState(cp.state) + cp.ParseModel() + return cp.cp.Result(tokens) +} + +func (p *CompletionParser) ParseModel() { + p.cp.EnterRule("Model", Model__Start) + defer p.cp.ExitRule() + { + p.cp.RecordSnapshot(Model__LoopEntry) + p.state.Sync(Model__LoopEntry) + for p.lookahead.ModelStatementsLoop(p.state) { + p.cp.MarkAssignment("Statements") + p.state.EnterRule(Model__Basic_1) + p.ParseStatement() + p.state.ExitRule() + p.cp.ClearAssignment() + p.cp.RecordSnapshot(Model__LoopEntry) + p.state.Sync(Model__LoopEntry) + } + } +} + +func (p *CompletionParser) ParseStatement() { + p.cp.EnterRule("Statement", Statement__Start) + defer p.cp.ExitRule() + { + p.state.EnterRule(Statement__Basic_1) + p.ParseVariableDecl() + p.state.ExitRule() + } +} + +func (p *CompletionParser) ParseVariableDecl() { + p.cp.EnterRule("VariableDecl", VariableDecl__Start) + defer p.cp.ExitRule() + { + p.cp.MarkAssignment("Name") + p.state.Consume(Token_ID) + p.cp.ClearAssignment() + } + { + p.state.Consume(Keyword_ColonEquals) + } + { + p.cp.MarkAssignment("Value") + p.state.EnterRule(VariableDecl__Basic_1) + p.ParseExpression() + p.state.ExitRule() + p.cp.ClearAssignment() + } +} + +func (p *CompletionParser) ParseExpression() { + p.cp.EnterRule("Expression", Expression__Start) + defer p.cp.ExitRule() + { + p.state.EnterRule(Expression__Basic_1) + p.ParseAdditive() + p.state.ExitRule() + } +} + +func (p *CompletionParser) ParseAdditive() { + p.cp.EnterRule("Additive", Additive__Start) + defer p.cp.ExitRule() + { + p.state.EnterRule(Additive__LoopEntry) + p.ParsePrimary() + p.state.ExitRule() + } + p.cp.RecordSnapshot(Additive__LoopEntry) + p.state.Sync(Additive__LoopEntry) + for p.lookahead.AdditiveLoop(p.state) { + { + p.cp.MarkAssignment("Operator") + p.state.Consume(Keyword_Plus) + p.cp.ClearAssignment() + } + { + p.cp.MarkAssignment("Right") + p.state.EnterRule(Additive__Basic_2) + p.ParsePrimary() + p.state.ExitRule() + p.cp.ClearAssignment() + } + p.cp.RecordSnapshot(Additive__LoopEntry) + p.state.Sync(Additive__LoopEntry) + } +} + +func (p *CompletionParser) ParsePrimary() { + p.cp.EnterRule("Primary", Primary__Start) + defer p.cp.ExitRule() + switch prediction, failure := p.lookahead.PrimaryAlternatives(p.state); prediction { + case 0: + { + p.state.EnterRule(Primary__Basic_1) + p.ParseVariableRef() + p.state.ExitRule() + } + case 1: + { + p.state.EnterRule(Primary__Basic_3) + p.ParseNumericLiteral() + p.state.ExitRule() + } + case 2: + { + p.state.EnterRule(Primary__Basic_5) + p.ParseStringLiteral() + p.state.ExitRule() + } + case 3: + { + p.state.EnterRule(Primary__Basic_7) + p.ParseParentheses() + p.state.ExitRule() + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } +} + +func (p *CompletionParser) ParseParentheses() { + p.cp.EnterRule("Parentheses", Parentheses__Start) + defer p.cp.ExitRule() + { + p.state.Consume(Keyword_LeftParen) + } + { + p.cp.MarkAssignment("Expr") + p.state.EnterRule(Parentheses_RightParen) + p.ParseExpression() + p.state.ExitRule() + p.cp.ClearAssignment() + } + { + p.state.Consume(Keyword_RightParen) + } +} + +func (p *CompletionParser) ParseVariableRef() { + p.cp.EnterRule("VariableRef", VariableRef__Start) + defer p.cp.ExitRule() + { + p.cp.MarkAssignment("Name") + p.state.Consume(Token_ID) + p.cp.ClearAssignment() + } +} + +func (p *CompletionParser) ParseNumericLiteral() { + p.cp.EnterRule("NumericLiteral", NumericLiteral__Start) + defer p.cp.ExitRule() + { + p.cp.MarkAssignment("Value") + p.state.Consume(Token_INT) + p.cp.ClearAssignment() + } +} + +func (p *CompletionParser) ParseStringLiteral() { + p.cp.EnterRule("StringLiteral", StringLiteral__Start) + defer p.cp.ExitRule() + { + p.state.Consume(Keyword_Backtick) + } + { + p.cp.RecordSnapshot(StringLiteral__LoopEntry) + p.state.Sync(StringLiteral__LoopEntry) + for p.lookahead.StringLiteralContentLoop(p.state) { + p.cp.MarkAssignment("Content") + p.state.EnterRule(StringLiteral__Basic_1) + p.ParseStringContent() + p.state.ExitRule() + p.cp.ClearAssignment() + p.cp.RecordSnapshot(StringLiteral__LoopEntry) + p.state.Sync(StringLiteral__LoopEntry) + } + } + { + p.state.Consume(Keyword_Backtick) + } +} + +func (p *CompletionParser) ParseStringContent() { + p.cp.EnterRule("StringContent", StringContent__Start) + defer p.cp.ExitRule() + switch prediction, failure := p.lookahead.StringContentAlternatives(p.state); prediction { + case 0: + { + p.state.EnterRule(StringContent__Basic_1) + p.ParseStringText() + p.state.ExitRule() + } + case 1: + { + p.state.EnterRule(StringContent__Basic_3) + p.ParseInterpolation() + p.state.ExitRule() + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } +} + +func (p *CompletionParser) ParseStringText() { + p.cp.EnterRule("StringText", StringText__Start) + defer p.cp.ExitRule() + { + p.cp.MarkAssignment("Value") + p.state.Consume(Token_STRING_CONTENT) + p.cp.ClearAssignment() + } +} + +func (p *CompletionParser) ParseInterpolation() { + p.cp.EnterRule("Interpolation", Interpolation__Start) + defer p.cp.ExitRule() + { + p.state.Consume(Keyword_HashLeftBrace) + } + { + p.cp.MarkAssignment("Expression") + p.state.EnterRule(Interpolation_RightBrace) + p.ParseExpression() + p.state.ExitRule() + p.cp.ClearAssignment() + } + { + p.state.Consume(Keyword_RightBrace) + } +} diff --git a/internal/languages/token_modes/lexer_gen.go b/internal/languages/token_modes/lexer_gen.go new file mode 100644 index 00000000..3ef8d222 --- /dev/null +++ b/internal/languages/token_modes/lexer_gen.go @@ -0,0 +1,673 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes + +import ( + "strings" + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/lexer" + "unicode/utf8" +) + +const Keyword_HashLeftBrace_Idx = 1 + +var Keyword_HashLeftBrace = core.NewTokenType( + Keyword_HashLeftBrace_Idx, + "#{", + "#{", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "#{") { + return 2 + } + return 0 + }, + []rune{'#'}, +) + +const Keyword_LeftParen_Idx = 2 + +var Keyword_LeftParen = core.NewTokenType( + Keyword_LeftParen_Idx, + "(", + "(", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "(") { + return 1 + } + return 0 + }, + []rune{'('}, +) + +const Keyword_RightParen_Idx = 3 + +var Keyword_RightParen = core.NewTokenType( + Keyword_RightParen_Idx, + ")", + ")", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], ")") { + return 1 + } + return 0 + }, + []rune{')'}, +) + +const Keyword_Plus_Idx = 4 + +var Keyword_Plus = core.NewTokenType( + Keyword_Plus_Idx, + "+", + "+", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "+") { + return 1 + } + return 0 + }, + []rune{'+'}, +) + +const Keyword_ColonEquals_Idx = 5 + +var Keyword_ColonEquals = core.NewTokenType( + Keyword_ColonEquals_Idx, + ":=", + ":=", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], ":=") { + return 2 + } + return 0 + }, + []rune{':'}, +) + +const Keyword_Backtick_Idx = 6 + +var Keyword_Backtick = core.NewTokenType( + Keyword_Backtick_Idx, + "`", + "`", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "`") { + return 1 + } + return 0 + }, + []rune{'`'}, +) + +const Keyword_RightBrace_Idx = 7 + +var Keyword_RightBrace = core.NewTokenType( + Keyword_RightBrace_Idx, + "}", + "}", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "}") { + return 1 + } + return 0 + }, + []rune{'}'}, +) + +const Token_ID_Idx = 8 + +var Token_ID = core.NewTokenType( + Token_ID_Idx, + "ID", + "ID", + core.TokenKindToken, + func(s string, offset int) int { + input := s[offset:] + length := len(input) + state := 0 + acceptedIndex := 0 + index := 0 + loop: + for index < length { + r, runeSize := utf8.DecodeRuneInString(input[index:]) + switch state { + case 0: + nextState := -1 + next := Token_ID_Next[0] + lookup := Token_ID_Lookup[0] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 1: + nextState := -1 + next := Token_ID_Next[1] + lookup := Token_ID_Lookup[1] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + default: + break loop + } + index += runeSize + if Token_ID_Accepting[state] { + acceptedIndex = index + } + } + return acceptedIndex + }, + []rune{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}, +) +var Token_ID_Lookup = [][]int64{ + {0x0000005A00000041, 0x0000005F0000005F, 0x0000007A00000061}, + {0x0000003900000030, 0x0000005A00000041, 0x0000005F0000005F, 0x0000007A00000061}, +} +var Token_ID_Next = [][]int{ + {1, 1, 1}, + {1, 1, 1, 1}, +} +var Token_ID_Accepting = [2]bool{ + 1: true, +} + +const Token_INT_Idx = 9 + +var Token_INT = core.NewTokenType( + Token_INT_Idx, + "INT", + "INT", + core.TokenKindToken, + func(s string, offset int) int { + input := s[offset:] + length := len(input) + state := 0 + acceptedIndex := 0 + index := 0 + loop: + for index < length { + r, runeSize := utf8.DecodeRuneInString(input[index:]) + switch state { + case 0: + nextState := -1 + next := Token_INT_Next[0] + lookup := Token_INT_Lookup[0] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 1: + nextState := -1 + next := Token_INT_Next[1] + lookup := Token_INT_Lookup[1] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + default: + break loop + } + index += runeSize + if Token_INT_Accepting[state] { + acceptedIndex = index + } + } + return acceptedIndex + }, + []rune{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}, +) +var Token_INT_Lookup = [][]int64{ + {0x0000003900000030}, + {0x0000003900000030}, +} +var Token_INT_Next = [][]int{ + {1}, + {1}, +} +var Token_INT_Accepting = [2]bool{ + 1: true, +} + +const Token_WS_Idx = 10 + +var Token_WS = core.NewTokenType( + Token_WS_Idx, + "WS", + "WS", + core.TokenKindToken, + func(s string, offset int) int { + input := s[offset:] + length := len(input) + state := 0 + acceptedIndex := 0 + index := 0 + loop: + for index < length { + r, runeSize := utf8.DecodeRuneInString(input[index:]) + switch state { + case 0: + nextState := -1 + next := Token_WS_Next[0] + lookup := Token_WS_Lookup[0] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 1: + nextState := -1 + next := Token_WS_Next[1] + lookup := Token_WS_Lookup[1] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + default: + break loop + } + index += runeSize + if Token_WS_Accepting[state] { + acceptedIndex = index + } + } + return acceptedIndex + }, + []rune{'\u0009', '\u000A', '\u000C', '\u000D', ' '}, +) +var Token_WS_Lookup = [][]int64{ + {0x0000000A00000009, 0x0000000D0000000C, 0x0000002000000020}, + {0x0000000A00000009, 0x0000000D0000000C, 0x0000002000000020}, +} +var Token_WS_Next = [][]int{ + {1, 1, 1}, + {1, 1, 1}, +} +var Token_WS_Accepting = [2]bool{ + 1: true, +} + +const Token_SL_COMMENT_Idx = 11 + +var Token_SL_COMMENT = core.NewTokenType( + Token_SL_COMMENT_Idx, + "SL_COMMENT", + "SL_COMMENT", + core.TokenKindToken, + func(s string, offset int) int { + input := s[offset:] + length := len(input) + state := 0 + acceptedIndex := 0 + index := 0 + loop: + for index < length { + r, runeSize := utf8.DecodeRuneInString(input[index:]) + switch state { + case 0: + nextState := -1 + next := Token_SL_COMMENT_Next[0] + lookup := Token_SL_COMMENT_Lookup[0] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 1: + nextState := -1 + next := Token_SL_COMMENT_Next[1] + lookup := Token_SL_COMMENT_Lookup[1] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 2: + nextState := -1 + next := Token_SL_COMMENT_Next[2] + lookup := Token_SL_COMMENT_Lookup[2] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + default: + break loop + } + index += runeSize + if Token_SL_COMMENT_Accepting[state] { + acceptedIndex = index + } + } + return acceptedIndex + }, + []rune{'/'}, +) +var Token_SL_COMMENT_Lookup = [][]int64{ + {0x0000002F0000002F}, + {0x0000002F0000002F}, + {0x0000000900000000, 0x0000000C0000000B, 0x0010FFFF0000000E}, +} +var Token_SL_COMMENT_Next = [][]int{ + {1}, + {2}, + {2, 2, 2}, +} +var Token_SL_COMMENT_Accepting = [3]bool{ + 2: true, +} + +const Token_ML_COMMENT_Idx = 12 + +var Token_ML_COMMENT = core.NewTokenType( + Token_ML_COMMENT_Idx, + "ML_COMMENT", + "ML_COMMENT", + core.TokenKindToken, + func(s string, offset int) int { + input := s[offset:] + length := len(input) + state := 0 + acceptedIndex := 0 + index := 0 + loop: + for index < length { + r, runeSize := utf8.DecodeRuneInString(input[index:]) + switch state { + case 0: + nextState := -1 + next := Token_ML_COMMENT_Next[0] + lookup := Token_ML_COMMENT_Lookup[0] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 1: + nextState := -1 + next := Token_ML_COMMENT_Next[1] + lookup := Token_ML_COMMENT_Lookup[1] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 2: + nextState := -1 + next := Token_ML_COMMENT_Next[2] + lookup := Token_ML_COMMENT_Lookup[2] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 3: + nextState := -1 + next := Token_ML_COMMENT_Next[3] + lookup := Token_ML_COMMENT_Lookup[3] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 4: + nextState := -1 + next := Token_ML_COMMENT_Next[4] + lookup := Token_ML_COMMENT_Lookup[4] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + default: + break loop + } + index += runeSize + if Token_ML_COMMENT_Accepting[state] { + acceptedIndex = index + } + } + return acceptedIndex + }, + []rune{'/'}, +) +var Token_ML_COMMENT_Lookup = [][]int64{ + {0x0000002F0000002F}, + {0x0000002A0000002A}, + {0x0000002900000000, 0x0000002A0000002A, 0x0010FFFF0000002B}, + {0x0000002900000000, 0x0000002A0000002A, 0x0000002E0000002B, 0x0000002F0000002F, 0x0010FFFF00000030}, + {}, +} +var Token_ML_COMMENT_Next = [][]int{ + {1}, + {2}, + {2, 3, 2}, + {2, 3, 2, 4, 2}, + {}, +} +var Token_ML_COMMENT_Accepting = [5]bool{ + 4: true, +} + +const Token_STRING_CONTENT_Idx = 13 + +var Token_STRING_CONTENT = core.NewTokenType( + Token_STRING_CONTENT_Idx, + "STRING_CONTENT", + "STRING_CONTENT", + core.TokenKindToken, + func(s string, offset int) int { + input := s[offset:] + length := len(input) + state := 0 + acceptedIndex := 0 + index := 0 + loop: + for index < length { + r, runeSize := utf8.DecodeRuneInString(input[index:]) + switch state { + case 0: + nextState := -1 + next := Token_STRING_CONTENT_Next[0] + lookup := Token_STRING_CONTENT_Lookup[0] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 1: + nextState := -1 + next := Token_STRING_CONTENT_Next[1] + lookup := Token_STRING_CONTENT_Lookup[1] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + case 2: + nextState := -1 + next := Token_STRING_CONTENT_Next[2] + lookup := Token_STRING_CONTENT_Lookup[2] + for i, lowHigh := range lookup { + if rune(lowHigh&0xFFFFFFFF) <= r && r <= rune(lowHigh>>32) { + nextState = next[i] + break + } + } + if nextState > -1 { + state = nextState + } else { + break loop + } + default: + break loop + } + index += runeSize + if Token_STRING_CONTENT_Accepting[state] { + acceptedIndex = index + } + } + return acceptedIndex + }, + []rune{'\u0000', '\u0001', '\u0002', '\u0003', '\u0004', '\u0005', '\u0006', '\u0007', '\u0008', '\u0009', '\u000A', '\u000B', '\u000C', '\u000D', '\u000E', '\u000F', '\u0010', '\u0011', '\u0012', '\u0013', '\u0014', '\u0015', '\u0016', '\u0017', '\u0018', '\u0019', '\u001A', '\u001B', '\u001C', '\u001D', '\u001E', '\u001F', ' ', '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', '\u007F', '\u0080', '\u0081', '\u0082', '\u0083', '\u0084', '\u0085', '\u0086', '\u0087', '\u0088', '\u0089', '\u008A', '\u008B', '\u008C', '\u008D', '\u008E', '\u008F', '\u0090', '\u0091', '\u0092', '\u0093', '\u0094', '\u0095', '\u0096', '\u0097', '\u0098', '\u0099', '\u009A', '\u009B', '\u009C', '\u009D', '\u009E', '\u009F', '\u00A0', '\u00A1', '\u00A2', '\u00A3', '\u00A4', '\u00A5', '\u00A6', '\u00A7', '\u00A8', '\u00A9', '\u00AA', '\u00AB', '\u00AC', '\u00AD', '\u00AE', '\u00AF', '\u00B0', '\u00B1', '\u00B2', '\u00B3', '\u00B4', '\u00B5', '\u00B6', '\u00B7', '\u00B8', '\u00B9', '\u00BA', '\u00BB', '\u00BC', '\u00BD', '\u00BE', '\u00BF', '\u00C0', '\u00C1', '\u00C2', '\u00C3', '\u00C4', '\u00C5', '\u00C6', '\u00C7', '\u00C8', '\u00C9', '\u00CA', '\u00CB', '\u00CC', '\u00CD', '\u00CE', '\u00CF', '\u00D0', '\u00D1', '\u00D2', '\u00D3', '\u00D4', '\u00D5', '\u00D6', '\u00D7', '\u00D8', '\u00D9', '\u00DA', '\u00DB', '\u00DC', '\u00DD', '\u00DE', '\u00DF', '\u00E0', '\u00E1', '\u00E2', '\u00E3', '\u00E4', '\u00E5', '\u00E6', '\u00E7', '\u00E8', '\u00E9', '\u00EA', '\u00EB', '\u00EC', '\u00ED', '\u00EE', '\u00EF', '\u00F0', '\u00F1', '\u00F2', '\u00F3', '\u00F4', '\u00F5', '\u00F6', '\u00F7', '\u00F8', '\u00F9', '\u00FA', '\u00FB', '\u00FC', '\u00FD', '\u00FE', '\u00FF'}, +) +var Token_STRING_CONTENT_Lookup = [][]int64{ + {0x0000002200000000, 0x0000005B00000024, 0x0000005C0000005C, 0x0000005F0000005D, 0x0010FFFF00000061}, + {0x0000002200000000, 0x0000005B00000024, 0x0000005C0000005C, 0x0000005F0000005D, 0x0010FFFF00000061}, + {0x0000002300000023, 0x0000005C0000005C, 0x0000006000000060, 0x0000006E0000006E, 0x0000007200000072, 0x0000007400000074}, +} +var Token_STRING_CONTENT_Next = [][]int{ + {1, 1, 2, 1, 1}, + {1, 1, 2, 1, 1}, + {1, 1, 1, 1, 1, 1}, +} +var Token_STRING_CONTENT_Accepting = [3]bool{ + 1: true, +} + +const ( + TokenMode_default = 0 + TokenMode_IN_STRING = 1 + TokenMode_IN_INTERPOLATION = 2 +) + +func NewLexer() lexer.Lexer { + modes := make([]*lexer.TokenMode, 3) + modes[TokenMode_default] = lexer.NewTokenMode("default", + lexer.UseTokenType(Keyword_ColonEquals), + lexer.UseTokenType(Keyword_Backtick).WithPushMode(TokenMode_IN_STRING), + lexer.UseTokenType(Keyword_Plus), + lexer.UseTokenType(Keyword_LeftParen), + lexer.UseTokenType(Keyword_RightParen), + lexer.UseTokenType(Token_ID), + lexer.UseTokenType(Token_INT), + lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier), + lexer.UseTokenType(Token_SL_COMMENT).WithModifier(core.CommentModifier), + lexer.UseTokenType(Token_ML_COMMENT).WithModifier(core.CommentModifier), + ) + modes[TokenMode_IN_STRING] = lexer.NewTokenMode("IN_STRING", + lexer.UseTokenType(Keyword_Backtick).WithPopMode(), + lexer.UseTokenType(Keyword_HashLeftBrace).WithPushMode(TokenMode_IN_INTERPOLATION), + lexer.UseTokenType(Token_STRING_CONTENT), + ) + modes[TokenMode_IN_INTERPOLATION] = lexer.NewTokenMode("IN_INTERPOLATION", + lexer.UseTokenType(Keyword_RightBrace).WithPopMode(), + lexer.UseTokenType(Keyword_Backtick).WithPushMode(TokenMode_IN_STRING), + lexer.UseTokenType(Keyword_Plus), + lexer.UseTokenType(Keyword_LeftParen), + lexer.UseTokenType(Keyword_RightParen), + lexer.UseTokenType(Token_ID), + lexer.UseTokenType(Token_INT), + lexer.UseTokenType(Token_WS).WithModifier(core.SkippedModifier), + lexer.UseTokenType(Token_SL_COMMENT).WithModifier(core.CommentModifier), + lexer.UseTokenType(Token_ML_COMMENT).WithModifier(core.CommentModifier), + ) + return lexer.NewDefaultLexer(TokenMode_default, modes...) +} diff --git a/internal/languages/token_modes/lexer_modes_test.go b/internal/languages/token_modes/lexer_modes_test.go new file mode 100644 index 00000000..6dce4809 --- /dev/null +++ b/internal/languages/token_modes/lexer_modes_test.go @@ -0,0 +1,276 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package token_modes + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "typefox.dev/fastbelt/test" +) + +// stringOf parses src and returns the string literal marked with the "string" label. +func stringOf(t *testing.T, src string) StringLiteral { + t.Helper() + fixture := test.New(t, CreateServices()) + doc := fixture.Parse(src) + doc.AssertNoErrors() + return test.MustFindNodeAtLabel[StringLiteral](doc, "string") +} + +// --- String content shapes --- + +func TestEmptyString(t *testing.T) { + str := stringOf(t, "VAR := <|string:``|>") + assert.Empty(t, str.Content()) +} + +func TestStringWithOnlyInterpolation(t *testing.T) { + str := stringOf(t, "NUM := 1\nVAR := <|string:`#{NUM}`|>") + content := str.Content() + require.Len(t, content, 1) + interpolation, ok := content[0].(Interpolation) + require.True(t, ok) + assert.Equal(t, "NUM", interpolation.Expression().(VariableRef).Name().Ref(context.Background()).Name()) +} + +func TestStringPreservesWhitespace(t *testing.T) { + // WS is skipped in the default mode but is not registered in IN_STRING, so + // whitespace inside string text must survive as part of STRING_CONTENT. + str := stringOf(t, "VAR := <|string:` spaced out `|>") + content := str.Content() + require.Len(t, content, 1) + assert.Equal(t, " spaced out ", content[0].(StringText).Value()) +} + +func TestStringPreservesCommentSyntax(t *testing.T) { + // Comments are only recognized in the default and interpolation modes, so + // `//` and `/* */` inside a string are plain text. + str := stringOf(t, "VAR := <|string:`// not a comment /* nor this */`|>") + content := str.Content() + require.Len(t, content, 1) + assert.Equal(t, "// not a comment /* nor this */", content[0].(StringText).Value()) +} + +func TestStringTextAroundInterpolation(t *testing.T) { + str := stringOf(t, "NUM := 1\nVAR := <|string:`a #{NUM} b`|>") + content := str.Content() + require.Len(t, content, 3) + assert.Equal(t, "a ", content[0].(StringText).Value()) + _, isInterpolation := content[1].(Interpolation) + assert.True(t, isInterpolation) + assert.Equal(t, " b", content[2].(StringText).Value()) +} + +// --- Interpolation mode --- + +func TestWhitespaceInsideInterpolationIsSkipped(t *testing.T) { + // WS is hidden in IN_INTERPOLATION, so padding around the expression does + // not become part of the AST. + str := stringOf(t, "NUM := 1\nVAR := <|string:`#{ NUM }`|>") + content := str.Content() + require.Len(t, content, 1) + interpolation := content[0].(Interpolation) + assert.Equal(t, "NUM", interpolation.Expression().(VariableRef).Name().Ref(context.Background()).Name()) +} + +func TestCommentInsideInterpolationIsIgnored(t *testing.T) { + str := stringOf(t, "NUM := 1\nVAR := <|string:`#{NUM /* here */}`|>") + content := str.Content() + require.Len(t, content, 1) + interpolation := content[0].(Interpolation) + assert.Equal(t, "NUM", interpolation.Expression().(VariableRef).Name().Ref(context.Background()).Name()) +} + +func TestExpressionInsideInterpolation(t *testing.T) { + str := stringOf(t, "NUM := 1\nVAR := <|string:`#{(NUM + 2) + 3}`|>") + content := str.Content() + require.Len(t, content, 1) + interpolation := content[0].(Interpolation) + _, isBinary := interpolation.Expression().(BinaryExpression) + assert.True(t, isBinary) +} + +func TestDeeplyNestedStringInterpolation(t *testing.T) { + // default -> IN_STRING -> IN_INTERPOLATION -> IN_STRING -> IN_INTERPOLATION + // -> IN_STRING, i.e. a mode stack six levels deep. + str := stringOf(t, "NUM := 1\nVAR := <|string:`a#{`b#{`c#{NUM}c`}b`}a`|>") + content := str.Content() + require.Len(t, content, 3) + assert.Equal(t, "a", content[0].(StringText).Value()) + assert.Equal(t, "a", content[2].(StringText).Value()) + + level2 := content[1].(Interpolation).Expression().(StringLiteral).Content() + require.Len(t, level2, 3) + assert.Equal(t, "b", level2[0].(StringText).Value()) + + level3 := level2[1].(Interpolation).Expression().(StringLiteral).Content() + require.Len(t, level3, 3) + assert.Equal(t, "c", level3[0].(StringText).Value()) + assert.Equal(t, + "NUM", + level3[1].(Interpolation).Expression().(VariableRef).Name().Ref(context.Background()).Name(), + ) +} + +func TestConsecutiveStringsUseIndependentModeStacks(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("A := `one`\nB := `two`\nC := `three`") + doc.AssertNoErrors() + strings := test.FindAll[StringLiteral](doc) + require.Len(t, strings, 3) + for i, expected := range []string{"one", "two", "three"} { + content := strings[i].Content() + require.Len(t, content, 1) + assert.Equal(t, expected, content[0].(StringText).Value()) + } +} + +// --- Cross references out of an interpolation --- + +func TestInterpolationReferencesVariableDeclaredLater(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("VAR := `#{LATER}`\nLATER := 42") + doc.AssertNoErrors() + ref := test.MustFindNode[Interpolation](doc) + target := ref.Expression().(VariableRef).Name().Ref(doc.Ctx()) + require.NotNil(t, target) + assert.Equal(t, "LATER", target.Name()) +} + +func TestInterpolationReferencesUnknownVariable(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("VAR := `#{MISSING}`") + // The lexer handles the mode switches; the unresolved name is a linking + // error, not a lexer error. + assert.Empty(t, doc.Document.LexerErrors) + assert.Empty(t, doc.Document.ParserErrors) + ref := test.MustFindNode[Interpolation](doc) + assert.NotNil(t, ref.Expression().(VariableRef).Name().Error()) +} + +// --- Malformed input --- + +func TestUnterminatedString(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("VAR := `unterminated") + // The lexer stays in IN_STRING to the end of input; the missing backtick is + // reported by the parser. + assert.Empty(t, doc.Document.LexerErrors) + assert.NotEmpty(t, doc.Document.ParserErrors) +} + +func TestUnterminatedInterpolation(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("NUM := 1\nVAR := `text #{NUM") + assert.Empty(t, doc.Document.LexerErrors) + assert.NotEmpty(t, doc.Document.ParserErrors) +} + +func TestUnterminatedStringDoesNotAffectFollowingDocument(t *testing.T) { + fixture := test.New(t, CreateServices()) + // Both documents share one lexer instance. The first one ends inside + // IN_STRING, which must not carry over. + docs := fixture.ParseAll( + "inmemory://broken.tm", "VAR := `unterminated", + "inmemory://valid.tm", "OTHER := 123", + ) + valid := docs[1] + assert.Empty(t, valid.Document.LexerErrors) + assert.Empty(t, valid.Document.ParserErrors) + decl := test.MustFindNode[VariableDecl](valid) + assert.Equal(t, "OTHER", decl.Name()) +} + +func TestStrayClosingBraceOutsideInterpolation(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("VAR := 1\n}") + // "}" is only registered in IN_INTERPOLATION, so in the default mode it is + // unrecognized input. + require.NotEmpty(t, doc.Document.LexerErrors) + assert.Equal(t, "No matching token", doc.Document.LexerErrors[0].Msg) +} + +func TestBareHashInsideStringIsUnrecognized(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("VAR := `a # b`") + // STRING_CONTENT excludes '#', and only "#{" enters the interpolation mode, + // so a lone '#' cannot be matched in IN_STRING. + require.NotEmpty(t, doc.Document.LexerErrors) + assert.Equal(t, "No matching token", doc.Document.LexerErrors[0].Msg) +} + +// bt is a single backtick, which delimits strings in this language. +const bt = "`" + +func TestEscapesInsideString(t *testing.T) { + // An escaped backtick or hash stays inside the string instead of switching + // modes. + str := stringOf(t, "VAR := <|string:"+bt+`esc\`+bt+` and \#{ ok`+bt+"|>") + content := str.Content() + require.Len(t, content, 1) + assert.Equal(t, `esc\`+bt+` and \#{ ok`, content[0].(StringText).Value()) +} + +func TestEscapedBackslashBeforeStringTerminator(t *testing.T) { + // STRING_CONTENT excludes the backslash from its literal alternative + // (/([^`#\\]|\\[`#tnr\\])+/), so a backslash can only ever start an escape. + // That keeps `\\` from consuming the closing backtick as `\` + `\` + "`", + // which would lose the terminator. + str := stringOf(t, "VAR := <|string:"+bt+`a\\`+bt+"|>") + content := str.Content() + require.Len(t, content, 1) + assert.Equal(t, `a\\`, content[0].(StringText).Value()) +} + +func TestEscapedBackslashFollowedByInterpolation(t *testing.T) { + // The same ambiguity would exist for "#{": `\\#{...}` must escape the + // backslash and then enter the interpolation mode. + str := stringOf(t, "NUM := 1\nVAR := <|string:"+bt+`a\\#{NUM}`+bt+"|>") + content := str.Content() + require.Len(t, content, 2) + assert.Equal(t, `a\\`, content[0].(StringText).Value()) + interpolation, ok := content[1].(Interpolation) + require.True(t, ok) + assert.Equal(t, "NUM", interpolation.Expression().(VariableRef).Name().Ref(context.Background()).Name()) +} + +func TestAllEscapeSequencesInsideString(t *testing.T) { + str := stringOf(t, "VAR := <|string:"+bt+`\`+bt+`\#\t\n\r\\`+bt+"|>") + content := str.Content() + require.Len(t, content, 1) + assert.Equal(t, `\`+bt+`\#\t\n\r\\`, content[0].(StringText).Value()) +} + +func TestUnknownEscapeInsideStringIsUnrecognized(t *testing.T) { + fixture := test.New(t, CreateServices()) + // Now that a lone backslash is not literal text, an escape outside + // [`#tnr\] cannot be matched at all. + doc := fixture.Parse("VAR := " + bt + `a\xb` + bt) + require.NotEmpty(t, doc.Document.LexerErrors) + assert.Equal(t, "No matching token", doc.Document.LexerErrors[0].Msg) + // The lexer recovers by dropping the offending rune and stays in IN_STRING, + // so the closing backtick still terminates the string. + assert.Empty(t, doc.Document.ParserErrors) +} + +func TestEscapedTerminatorLeavesStringUnterminated(t *testing.T) { + fixture := test.New(t, CreateServices()) + // A single backslash before the closing backtick escapes it, so the whole + // input is one unterminated string rather than a lexer error. + doc := fixture.Parse("VAR := " + bt + `a\` + bt) + assert.Empty(t, doc.Document.LexerErrors) + assert.NotEmpty(t, doc.Document.ParserErrors) + + // Adding a real terminator closes the string, with the escape kept as text. + closed := test.New(t, CreateServices()).Parse("VAR := <|string:" + bt + `a\` + bt + bt + "|>") + closed.AssertNoErrors() + node := test.MustFindNodeAtLabel[StringLiteral](closed, "string") + content := node.Content() + require.Len(t, content, 1) + assert.Equal(t, `a\`+bt, content[0].(StringText).Value()) +} diff --git a/internal/languages/token_modes/linker_gen.go b/internal/languages/token_modes/linker_gen.go new file mode 100644 index 00000000..29fcd644 --- /dev/null +++ b/internal/languages/token_modes/linker_gen.go @@ -0,0 +1,116 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes + +import ( + "context" + "reflect" + "slices" + "sync" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/linking" + "typefox.dev/fastbelt/util/extiter" + "typefox.dev/fastbelt/util/service" +) + +type TokenModesScopeProvider interface { + ScopeVariableRefName(ctx context.Context, reference *core.Reference[VariableDecl]) core.Scope +} + +type DefaultTokenModesScopeProvider struct { + sc *service.Container +} + +func NewDefaultTokenModesScopeProvider(sc *service.Container) TokenModesScopeProvider { + return &DefaultTokenModesScopeProvider{sc: sc} +} + +func (s *DefaultTokenModesScopeProvider) ScopeVariableRefName(ctx context.Context, reference *core.Reference[VariableDecl]) core.Scope { + return linking.DefaultScopeOfType[VariableDecl](reference.Owner()) +} + +type TokenModesReferenceLinker interface { + LinkVariableRefName(ctx context.Context, reference *core.Reference[VariableDecl]) (*core.SymbolDescription, *core.ReferenceError) +} + +type DefaultTokenModesReferenceLinker struct { + sc *service.Container + scopeProvider func() TokenModesScopeProvider +} + +func NewDefaultTokenModesReferenceLinker(sc *service.Container) TokenModesReferenceLinker { + return &DefaultTokenModesReferenceLinker{ + sc: sc, + scopeProvider: sync.OnceValue(func() TokenModesScopeProvider { + return service.MustGet[TokenModesScopeProvider](sc) + }), + } +} + +func (s *DefaultTokenModesReferenceLinker) LinkVariableRefName(ctx context.Context, reference *core.Reference[VariableDecl]) (*core.SymbolDescription, *core.ReferenceError) { + scope := s.scopeProvider().ScopeVariableRefName(ctx, reference) + return core.DefaultLink(scope, reference.Text()) +} + +type TokenModesReferencesConstructor interface { + VariableRefName(owner core.AstNode, unit core.StringUnit) *core.Reference[VariableDecl] +} + +type DefaultTokenModesReferencesConstructor struct { + sc *service.Container + referenceLinker func() TokenModesReferenceLinker +} + +func NewDefaultTokenModesReferencesConstructor(sc *service.Container) TokenModesReferencesConstructor { + return &DefaultTokenModesReferencesConstructor{ + sc: sc, + referenceLinker: sync.OnceValue(func() TokenModesReferenceLinker { + return service.MustGet[TokenModesReferenceLinker](sc) + }), + } +} + +func (s *DefaultTokenModesReferencesConstructor) VariableRefName(owner core.AstNode, unit core.StringUnit) *core.Reference[VariableDecl] { + fn := s.referenceLinker().LinkVariableRefName + return core.NewReference(owner, unit, fn) +} + +type TokenModesSymbolContainers struct{} + +func (c *TokenModesSymbolContainers) New() core.SymbolContainer { + return &TokenModesSymbolContainer{} +} + +func NewSymbolContainers() *TokenModesSymbolContainers { + return &TokenModesSymbolContainers{} +} + +type TokenModesSymbolContainer struct { + VariableDecls []*core.SymbolDescription +} + +func (sc *TokenModesSymbolContainer) Put(desc *core.SymbolDescription) bool { + switch desc.Node.(type) { + case VariableDecl: + sc.VariableDecls = append(sc.VariableDecls, desc) + return true + } + return false +} + +func (sc *TokenModesSymbolContainer) All() core.SymbolSeq { + return extiter.Concat( + slices.Values(sc.VariableDecls), + ) +} + +var TypeFor_VariableDecl = reflect.TypeFor[VariableDecl]() + +func (sc *TokenModesSymbolContainer) ForType(t reflect.Type) core.SymbolSeq { + switch t { + case TypeFor_VariableDecl: + return slices.Values(sc.VariableDecls) + } + return core.EmptySymbolDescriptions +} diff --git a/internal/languages/token_modes/parser_gen.go b/internal/languages/token_modes/parser_gen.go new file mode 100644 index 00000000..4c3713d1 --- /dev/null +++ b/internal/languages/token_modes/parser_gen.go @@ -0,0 +1,354 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes + +import ( + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/parser" + "typefox.dev/fastbelt/util/service" +) + +type Parser struct { + state *parser.ParserState + sc *service.Container + referencesConstructor TokenModesReferencesConstructor + lookahead TokenModesParserLookahead +} + +func (p *Parser) Parse(document *core.Document) *parser.ParseResult { + recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) + messages := service.MustGet[parser.ErrorMessageProvider](p.sc) + referencesConstructor := service.MustGet[TokenModesReferencesConstructor](p.sc) + lookahead := service.MustGet[TokenModesParserLookahead](p.sc) + cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)} + result := cp.ParseModel() + cp.state.ExpectEndOfInput() + return &parser.ParseResult{Node: result, Errors: cp.state.Errors()} +} + +func NewParser(sc *service.Container) *Parser { + return &Parser{ + sc: sc, + } +} + +func (p *Parser) ParseModel() Model { + current := NewModel() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + p.state.Sync(Model__LoopEntry) + for p.lookahead.ModelStatementsLoop(p.state) { + p.state.EnterRule(Model__Basic_1) + result := p.ParseStatement() + p.state.ExitRule() + if result != nil { + current.SetStatementsItem(result) + } + p.state.Sync(Model__LoopEntry) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseStatement() Statement { + var current Statement + { + { + p.state.EnterRule(Statement__Basic_1) + result := p.ParseVariableDecl() + p.state.ExitRule() + current = result + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseVariableDecl() VariableDecl { + current := NewVariableDecl() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Token_ID) + core.AssignToken(current, token, VariableDecl_Name_ID) + if token != nil { + current.SetName(token) + } + } + { + token := p.state.Consume(Keyword_ColonEquals) + core.AssignToken(current, token, VariableDecl_ColonEquals) + } + { + p.state.EnterRule(VariableDecl__Basic_1) + result := p.ParseExpression() + p.state.ExitRule() + if result != nil { + current.SetValue(result) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseExpression() Expression { + var current Expression + { + { + p.state.EnterRule(Expression__Basic_1) + result := p.ParseAdditive() + p.state.ExitRule() + current = result + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseAdditive() Expression { + var current Expression + { + { + p.state.EnterRule(Additive__LoopEntry) + result := p.ParsePrimary() + p.state.ExitRule() + current = result + } + p.state.Sync(Additive__LoopEntry) + for p.lookahead.AdditiveLoop(p.state) { + { + result := NewBinaryExpression() + result.SetTextRange(current.TextRange()) + result.SetLeft(current) + current.SetTextRangeEnd(p.state.LA(0).Range.End) + current = result + } + current := current.(BinaryExpression) + { + token := p.state.Consume(Keyword_Plus) + core.AssignToken(current, token, Additive_Operator_Plus) + if token != nil { + current.SetOperator(token) + } + } + { + p.state.EnterRule(Additive__Basic_2) + result := p.ParsePrimary() + p.state.ExitRule() + if result != nil { + current.SetRight(result) + } + } + p.state.Sync(Additive__LoopEntry) + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParsePrimary() Primary { + startPos := p.state.LA(1).Range.Start + var current Primary + { + switch prediction, failure := p.lookahead.PrimaryAlternatives(p.state); prediction { + case 0: + { + p.state.EnterRule(Primary__Basic_1) + result := p.ParseVariableRef() + p.state.ExitRule() + current = result + } + case 1: + { + p.state.EnterRule(Primary__Basic_3) + result := p.ParseNumericLiteral() + p.state.ExitRule() + current = result + } + case 2: + { + p.state.EnterRule(Primary__Basic_5) + result := p.ParseStringLiteral() + p.state.ExitRule() + current = result + } + case 3: + { + p.state.EnterRule(Primary__Basic_7) + result := p.ParseParentheses() + p.state.ExitRule() + current = result + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } + if current == nil { + current = NewPrimary() + current.SetTextRangeStart(startPos) + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseParentheses() Parentheses { + current := NewParentheses() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Keyword_LeftParen) + core.AssignToken(current, token, Parentheses_LeftParen) + } + { + p.state.EnterRule(Parentheses_RightParen) + result := p.ParseExpression() + p.state.ExitRule() + if result != nil { + current.SetExpr(result) + } + } + { + token := p.state.Consume(Keyword_RightParen) + core.AssignToken(current, token, Parentheses_RightParen) + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseVariableRef() VariableRef { + current := NewVariableRef() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Token_ID) + core.AssignToken(current, token, VariableRef_Name_ID) + if token != nil { + current.SetName(p.referencesConstructor.VariableRefName(current, token)) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseNumericLiteral() NumericLiteral { + current := NewNumericLiteral() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Token_INT) + core.AssignToken(current, token, NumericLiteral_Value_INT) + if token != nil { + current.SetValue(token) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseStringLiteral() StringLiteral { + current := NewStringLiteral() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Keyword_Backtick) + core.AssignToken(current, token, StringLiteral_Backtick_0) + } + { + p.state.Sync(StringLiteral__LoopEntry) + for p.lookahead.StringLiteralContentLoop(p.state) { + p.state.EnterRule(StringLiteral__Basic_1) + result := p.ParseStringContent() + p.state.ExitRule() + if result != nil { + current.SetContentItem(result) + } + p.state.Sync(StringLiteral__LoopEntry) + } + } + { + token := p.state.Consume(Keyword_Backtick) + core.AssignToken(current, token, StringLiteral_Backtick_1) + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseStringContent() StringContent { + startPos := p.state.LA(1).Range.Start + var current StringContent + { + switch prediction, failure := p.lookahead.StringContentAlternatives(p.state); prediction { + case 0: + { + p.state.EnterRule(StringContent__Basic_1) + result := p.ParseStringText() + p.state.ExitRule() + current = result + } + case 1: + { + p.state.EnterRule(StringContent__Basic_3) + result := p.ParseInterpolation() + p.state.ExitRule() + current = result + } + default: + p.state.AppendError(p.state.Messages().NoViableAlternative(failure), failure.Token) + } + if current == nil { + current = NewStringContent() + current.SetTextRangeStart(startPos) + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseStringText() StringText { + current := NewStringText() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Token_STRING_CONTENT) + core.AssignToken(current, token, StringText_Value_STRING_CONTENT) + if token != nil { + current.SetValue(token) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} + +func (p *Parser) ParseInterpolation() Interpolation { + current := NewInterpolation() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Keyword_HashLeftBrace) + core.AssignToken(current, token, Interpolation_HashLeftBrace) + } + { + p.state.EnterRule(Interpolation_RightBrace) + result := p.ParseExpression() + p.state.ExitRule() + if result != nil { + current.SetExpression(result) + } + } + { + token := p.state.Consume(Keyword_RightBrace) + core.AssignToken(current, token, Interpolation_RightBrace) + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} diff --git a/internal/languages/token_modes/parser_lookahead_gen.go b/internal/languages/token_modes/parser_lookahead_gen.go new file mode 100644 index 00000000..6e0a87b3 --- /dev/null +++ b/internal/languages/token_modes/parser_lookahead_gen.go @@ -0,0 +1,68 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes + +import ( + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/parser" +) + +var PrimaryAlternatives = parser.LL1Lookahead{ + Types: []*core.TokenType{Token_ID, Token_INT, Keyword_Backtick, Keyword_LeftParen}, + Lookup: []int{2: 4, 6: 3, 8: 1, 9: 2}, +} + +var StringContentAlternatives = parser.LL1Lookahead{ + Types: []*core.TokenType{Token_STRING_CONTENT, Keyword_HashLeftBrace}, + Lookup: []int{1: 2, 13: 1}, +} + +var StringLiteralContentLoop = parser.LL1Lookahead{ + Types: []*core.TokenType{Keyword_HashLeftBrace, Token_STRING_CONTENT}, + Lookup: []int{1: 1, 13: 1}, +} + +// TokenModesParserLookahead abstracts every lookahead/prediction decision performed by +// the generated parser. Each method corresponds to a single decision point; +// implementations may override individual decisions while delegating the rest +// to DefaultTokenModesParserLookahead. +type TokenModesParserLookahead interface { + parser.ParserLookahead + + AdditiveLoop(state *parser.ParserState) bool + ModelStatementsLoop(state *parser.ParserState) bool + PrimaryAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + StringContentAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) + StringLiteralContentLoop(state *parser.ParserState) bool +} + +// DefaultTokenModesParserLookahead resolves every decision with the parser state's built-in +// static LL(1) and adaptive ALL(*) prediction. +type DefaultTokenModesParserLookahead struct { + parser.DefaultParserLookahead +} + +func NewDefaultTokenModesParserLookahead() TokenModesParserLookahead { + return &DefaultTokenModesParserLookahead{} +} + +func (l *DefaultTokenModesParserLookahead) AdditiveLoop(state *parser.ParserState) bool { + return state.LA(1).Type == Keyword_Plus +} + +func (l *DefaultTokenModesParserLookahead) ModelStatementsLoop(state *parser.ParserState) bool { + return state.LA(1).Type == Token_ID +} + +func (l *DefaultTokenModesParserLookahead) PrimaryAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { + return state.Lookahead(PrimaryAlternatives) +} + +func (l *DefaultTokenModesParserLookahead) StringContentAlternatives(state *parser.ParserState) (int, *parser.PredictionFailure) { + return state.Lookahead(StringContentAlternatives) +} + +func (l *DefaultTokenModesParserLookahead) StringLiteralContentLoop(state *parser.ParserState) bool { + prediction, _ := state.Lookahead(StringLiteralContentLoop) + return prediction == 0 +} diff --git a/internal/languages/token_modes/parser_test.go b/internal/languages/token_modes/parser_test.go new file mode 100644 index 00000000..ee41b956 --- /dev/null +++ b/internal/languages/token_modes/parser_test.go @@ -0,0 +1,78 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package token_modes + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "typefox.dev/fastbelt/test" +) + +func TestDeclaration(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("VAR := 123") + doc.AssertNoErrors() +} + +func TestSimpleString(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("VAR := <|string:`simple string`|>") + doc.AssertNoErrors() + node := test.MustFindNodeAtLabel[StringLiteral](doc, "string") + str := node + content := str.Content() + require.Len(t, content, 1) + require.Equal(t, "simple string", content[0].(StringText).Value()) +} + +func TestEscapedString(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("VAR := <|string:`escaped\\nstring`|>") + doc.AssertNoErrors() + node := test.MustFindNodeAtLabel[StringLiteral](doc, "string") + content := node.Content() + require.Len(t, content, 1) + require.Equal(t, "escaped\\nstring", content[0].(StringText).Value()) +} + +func TestInterpolatingString(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("NUM := 123\nVAR := <|string:`nested #{NUM}`|>") + doc.AssertNoErrors() + node := test.MustFindNodeAtLabel[StringLiteral](doc, "string") + content := node.Content() + require.Len(t, content, 2) + content0, ok := content[0].(StringText) + require.True(t, ok) + require.Equal(t, "nested ", content0.Value()) + content1, ok := content[1].(Interpolation) + require.True(t, ok) + require.Equal(t, "NUM", content1.Expression().(VariableRef).Name().Ref(context.Background()).Name()) +} + +func TestNestedString(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("NUM := 123\nVAR := <|string:`this is #{`a number: #{NUM}`}`|>") + doc.AssertNoErrors() + node := test.MustFindNodeAtLabel[StringLiteral](doc, "string") + content := node.Content() + require.Len(t, content, 2) + content0, ok := content[0].(StringText) + require.True(t, ok) + require.Equal(t, "this is ", content0.Value()) + content1, ok := content[1].(Interpolation) + require.True(t, ok) + content2, ok := content1.Expression().(StringLiteral) + require.True(t, ok) + require.Len(t, content2.Content(), 2) + content20, ok := content2.Content()[0].(StringText) + require.True(t, ok) + require.Equal(t, "a number: ", content20.Value()) + content21, ok := content2.Content()[1].(Interpolation) + require.True(t, ok) + require.Equal(t, "NUM", content21.Expression().(VariableRef).Name().Ref(context.Background()).Name()) +} diff --git a/internal/languages/token_modes/services.go b/internal/languages/token_modes/services.go new file mode 100644 index 00000000..243f3adf --- /dev/null +++ b/internal/languages/token_modes/services.go @@ -0,0 +1,33 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package token_modes + +//go:generate go run ../../../cmd/fastbelt generate ./token_modes.fb -v + +import ( + "typefox.dev/fastbelt/linking" + "typefox.dev/fastbelt/server" + "typefox.dev/fastbelt/textdoc" + "typefox.dev/fastbelt/util/service" + "typefox.dev/fastbelt/workspace" +) + +func SetupServices(sc *service.Container) { + service.Put[workspace.LanguageID](sc, "modes") + service.Put[workspace.FileExtensions](sc, []string{".mode"}) + textdoc.SetupDefaultServices(sc) + linking.SetupDefaultServices(sc) + workspace.SetupDefaultServices(sc) + SetupGeneratedServices(sc) + SetupGeneratedServerServices(sc) + server.SetupDefaultServices(sc) +} + +func CreateServices() *service.Container { + sc := service.NewContainer() + SetupServices(sc) + sc.Seal() + return sc +} diff --git a/internal/languages/token_modes/services_gen.go b/internal/languages/token_modes/services_gen.go new file mode 100644 index 00000000..41c788c5 --- /dev/null +++ b/internal/languages/token_modes/services_gen.go @@ -0,0 +1,53 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes + +import ( + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/lexer" + "typefox.dev/fastbelt/parser" + "typefox.dev/fastbelt/util/service" +) + +// SetupGeneratedServices sets up the generated services for this grammar. +// If any service is already set, it's not overwritten. +func SetupGeneratedServices(sc *service.Container) { + if !service.Has[TokenModesScopeProvider](sc) { + service.Put(sc, NewDefaultTokenModesScopeProvider(sc)) + } + if !service.Has[TokenModesReferenceLinker](sc) { + service.Put(sc, NewDefaultTokenModesReferenceLinker(sc)) + } + if !service.Has[TokenModesReferencesConstructor](sc) { + service.Put(sc, NewDefaultTokenModesReferencesConstructor(sc)) + } + if !service.Has[TokenModesParserLookahead](sc) { + service.Put(sc, NewDefaultTokenModesParserLookahead()) + } + if !service.Has[lexer.Lexer](sc) { + service.Put(sc, NewLexer()) + } + if !service.Has[parser.Parser](sc) { + service.Put[parser.Parser](sc, NewParser(sc)) + } + if !service.Has[parser.ErrorRecoveryStrategy](sc) { + service.Put[parser.ErrorRecoveryStrategy](sc, parser.NewDefaultErrorRecovery()) + } + if !service.Has[parser.ErrorMessageProvider](sc) { + service.Put[parser.ErrorMessageProvider](sc, parser.NewDefaultErrorMessageProvider()) + } + if !service.Has[core.SymbolContainers](sc) { + service.Put[core.SymbolContainers](sc, NewSymbolContainers()) + } +} + +// SetupGeneratedServerServices sets up the generated language server services for this grammar. +// If any service is already set, it's not overwritten. +func SetupGeneratedServerServices(sc *service.Container) { + if !service.Has[TokenModesCompletionFilter](sc) { + service.Put(sc, NewDefaultTokenModesCompletionFilter()) + } + if !service.Has[parser.LanguageCompletionAdapter](sc) { + service.Put[parser.LanguageCompletionAdapter](sc, NewTokenModesCompletionAdapter(sc)) + } +} diff --git a/internal/languages/token_modes/token_modes.fb b/internal/languages/token_modes/token_modes.fb new file mode 100644 index 00000000..c45fe5e9 --- /dev/null +++ b/internal/languages/token_modes/token_modes.fb @@ -0,0 +1,119 @@ +grammar TokenModes + +interface Model { + Statements []Statement +} + +entry Model: + Statements+=Statement* + + +interface Statement {} + +Statement: VariableDecl + +interface VariableDecl extends Statement { + Name string + Value Expression +} + +VariableDecl: Name=ID ":=" Value=Expression + +interface Expression {} + +Expression: Additive; + +interface BinaryExpression extends Expression { + Left Expression + Operator string + Right Expression +} + +Additive returns Expression: Primary ({BinaryExpression.Left=current} Operator="+" Right=Primary)* + +interface Primary extends Expression {} + +Primary: + VariableRef + | NumericLiteral + | StringLiteral + | Parentheses + +interface Parentheses extends Primary { + Expr Expression +} + +Parentheses: "(" Expr=Expression ")" + +interface VariableRef extends Primary { + Name *VariableDecl +} + +VariableRef: Name=[VariableDecl:ID] + +interface NumericLiteral extends Primary { + Value string +} + +NumericLiteral: Value=INT + +interface StringLiteral extends Primary { + Content []StringContent +} + +StringLiteral: "`" Content+=StringContent* "`" + +interface StringContent {} + +StringContent: StringText | Interpolation + +interface StringText extends StringContent { + Value string +} + +StringText: Value=STRING_CONTENT + +interface Interpolation extends StringContent { + Expression Expression +} +Interpolation: "#{" Expression=Expression "}" + +token ID: /[a-zA-Z_][a-zA-Z0-9_]*/ +token INT: /[0-9]+/ +token WS: /\s+/ +token SL_COMMENT: /\/\/[^\r\n]*/ +token ML_COMMENT: /\/\*[\s\S]*?\*\// + +token mode default { + ":=" + + "`" -> push(IN_STRING) + ID + INT + "+" + "(" + ")" + hidden WS + comment SL_COMMENT + comment ML_COMMENT +} + +token mode IN_STRING { + "`" -> pop + token STRING_CONTENT: /([^`#\\]|\\[`#tnr\\])+/ + "#{" -> push(IN_INTERPOLATION) +} + +token mode IN_INTERPOLATION { + "}" -> pop + + "`" -> push(IN_STRING) + ID + INT + "+" + "(" + ")" + hidden WS + comment SL_COMMENT + comment ML_COMMENT +} diff --git a/internal/languages/token_modes/types_gen.go b/internal/languages/token_modes/types_gen.go new file mode 100644 index 00000000..a7b14179 --- /dev/null +++ b/internal/languages/token_modes/types_gen.go @@ -0,0 +1,955 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes + +import ( + "fmt" + "unique" + + core "typefox.dev/fastbelt" +) + +type Model interface { + core.AstNode + + IsModel() + Statements() []Statement + SetStatementsItem(item Statement) +} + +func NewModel() Model { + return &ModelImpl{} +} + +type ModelData struct { + statements []Statement +} + +func (i *ModelData) IsModel() {} + +func (i *ModelData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + for j, item := range i.statements { + fn(item, fieldNameStatements, j) + } +} + +func (i *ModelData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *ModelData) Statements() []Statement { + return i.statements +} + +func (i *ModelData) SetStatementsItem(item Statement) { + i.statements = append(i.statements, item) +} + +type ModelImpl struct { + core.AstNodeBase + ModelData +} + +func (i *ModelImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.ModelData.ForEachNode(fn) +} + +func (i *ModelImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.ModelData.ForEachReference(fn) +} + +func (i *ModelImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, index := path.Head() + switch field { + case fieldNameStatements: + if index >= len(i.Statements()) { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("ModelImpl.Resolve: index %d exceeds length of slice in 'statements' (length=%d) in node '%s'", index, len(i.Statements()), nodePath) + } + child := i.Statements()[index] + if child == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("ModelImpl.Resolve: item %d of slice in field 'statements' is nil in node '%s'", index, nodePath) + } + return child.Resolve(path.Tail()) + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("ModelImpl.Resolve: field '%s' does not exist in node '%s' of type 'Model'", field.Value(), nodePath) + } +} + +type Statement interface { + core.AstNode + + IsStatement() +} + +func NewStatement() Statement { + return &StatementImpl{} +} + +type StatementData struct { +} + +func (i *StatementData) IsStatement() {} + +func (i *StatementData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *StatementData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +type StatementImpl struct { + core.AstNodeBase + StatementData +} + +func (i *StatementImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.StatementData.ForEachNode(fn) +} + +func (i *StatementImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.StatementData.ForEachReference(fn) +} + +func (i *StatementImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("StatementImpl.Resolve: field '%s' does not exist in node '%s' of type 'Statement'", field.Value(), nodePath) +} + +type VariableDecl interface { + core.AstNode + Statement + + IsVariableDecl() + Name() string + NameToken() *core.Token + SetName(value *core.Token) + Value() Expression + SetValue(value Expression) +} + +func NewVariableDecl() VariableDecl { + return &VariableDeclImpl{} +} + +type VariableDeclData struct { + name *core.Token + value Expression +} + +func (i *VariableDeclData) IsVariableDecl() {} + +func (i *VariableDeclData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.value != nil { + fn(i.value, fieldNameValue, -1) + } +} + +func (i *VariableDeclData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *VariableDeclData) Name() string { + if i != nil && i.name != nil { + return i.name.Image + } else { + return "" + } +} + +func (i *VariableDeclData) NameToken() *core.Token { + return i.name +} + +func (i *VariableDeclData) SetName(value *core.Token) { + i.name = value +} + +func (i *VariableDeclData) Value() Expression { + if i != nil && i.value != nil { + return i.value + } else { + return nil + } +} + +func (i *VariableDeclData) SetValue(value Expression) { + i.value = value +} + +type VariableDeclImpl struct { + core.AstNodeBase + StatementData + VariableDeclData +} + +func (i *VariableDeclImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.StatementData.ForEachNode(fn) + i.VariableDeclData.ForEachNode(fn) +} + +func (i *VariableDeclImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.StatementData.ForEachReference(fn) + i.VariableDeclData.ForEachReference(fn) +} + +func (i *VariableDeclImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameValue: + if i.Value() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("VariableDeclImpl.Resolve: field 'value' is nil in node '%s'", nodePath) + } + child := i.Value() + return child.Resolve(path.Tail()) + case fieldNameName: + return nil, fmt.Errorf("VariableDeclImpl.Resolve: field 'name' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("VariableDeclImpl.Resolve: field '%s' does not exist in node '%s' of type 'VariableDecl'", field.Value(), nodePath) + } +} + +type Expression interface { + core.AstNode + + IsExpression() +} + +func NewExpression() Expression { + return &ExpressionImpl{} +} + +type ExpressionData struct { +} + +func (i *ExpressionData) IsExpression() {} + +func (i *ExpressionData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *ExpressionData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +type ExpressionImpl struct { + core.AstNodeBase + ExpressionData +} + +func (i *ExpressionImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.ExpressionData.ForEachNode(fn) +} + +func (i *ExpressionImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.ExpressionData.ForEachReference(fn) +} + +func (i *ExpressionImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("ExpressionImpl.Resolve: field '%s' does not exist in node '%s' of type 'Expression'", field.Value(), nodePath) +} + +type BinaryExpression interface { + core.AstNode + Expression + + IsBinaryExpression() + Left() Expression + SetLeft(value Expression) + Operator() string + OperatorToken() *core.Token + SetOperator(value *core.Token) + Right() Expression + SetRight(value Expression) +} + +func NewBinaryExpression() BinaryExpression { + return &BinaryExpressionImpl{} +} + +type BinaryExpressionData struct { + left Expression + operator *core.Token + right Expression +} + +func (i *BinaryExpressionData) IsBinaryExpression() {} + +func (i *BinaryExpressionData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.left != nil { + fn(i.left, fieldNameLeft, -1) + } + if i.right != nil { + fn(i.right, fieldNameRight, -1) + } +} + +func (i *BinaryExpressionData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *BinaryExpressionData) Left() Expression { + if i != nil && i.left != nil { + return i.left + } else { + return nil + } +} + +func (i *BinaryExpressionData) SetLeft(value Expression) { + i.left = value +} + +func (i *BinaryExpressionData) Operator() string { + if i != nil && i.operator != nil { + return i.operator.Image + } else { + return "" + } +} + +func (i *BinaryExpressionData) OperatorToken() *core.Token { + return i.operator +} + +func (i *BinaryExpressionData) SetOperator(value *core.Token) { + i.operator = value +} + +func (i *BinaryExpressionData) Right() Expression { + if i != nil && i.right != nil { + return i.right + } else { + return nil + } +} + +func (i *BinaryExpressionData) SetRight(value Expression) { + i.right = value +} + +type BinaryExpressionImpl struct { + core.AstNodeBase + ExpressionData + BinaryExpressionData +} + +func (i *BinaryExpressionImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.ExpressionData.ForEachNode(fn) + i.BinaryExpressionData.ForEachNode(fn) +} + +func (i *BinaryExpressionImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.ExpressionData.ForEachReference(fn) + i.BinaryExpressionData.ForEachReference(fn) +} + +func (i *BinaryExpressionImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameLeft: + if i.Left() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("BinaryExpressionImpl.Resolve: field 'left' is nil in node '%s'", nodePath) + } + child := i.Left() + return child.Resolve(path.Tail()) + case fieldNameRight: + if i.Right() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("BinaryExpressionImpl.Resolve: field 'right' is nil in node '%s'", nodePath) + } + child := i.Right() + return child.Resolve(path.Tail()) + case fieldNameOperator: + return nil, fmt.Errorf("BinaryExpressionImpl.Resolve: field 'operator' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("BinaryExpressionImpl.Resolve: field '%s' does not exist in node '%s' of type 'BinaryExpression'", field.Value(), nodePath) + } +} + +type Primary interface { + core.AstNode + Expression + + IsPrimary() +} + +func NewPrimary() Primary { + return &PrimaryImpl{} +} + +type PrimaryData struct { +} + +func (i *PrimaryData) IsPrimary() {} + +func (i *PrimaryData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *PrimaryData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +type PrimaryImpl struct { + core.AstNodeBase + ExpressionData + PrimaryData +} + +func (i *PrimaryImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.ExpressionData.ForEachNode(fn) + i.PrimaryData.ForEachNode(fn) +} + +func (i *PrimaryImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.ExpressionData.ForEachReference(fn) + i.PrimaryData.ForEachReference(fn) +} + +func (i *PrimaryImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("PrimaryImpl.Resolve: field '%s' does not exist in node '%s' of type 'Primary'", field.Value(), nodePath) +} + +type Parentheses interface { + core.AstNode + Primary + + IsParentheses() + Expr() Expression + SetExpr(value Expression) +} + +func NewParentheses() Parentheses { + return &ParenthesesImpl{} +} + +type ParenthesesData struct { + expr Expression +} + +func (i *ParenthesesData) IsParentheses() {} + +func (i *ParenthesesData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.expr != nil { + fn(i.expr, fieldNameExpr, -1) + } +} + +func (i *ParenthesesData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *ParenthesesData) Expr() Expression { + if i != nil && i.expr != nil { + return i.expr + } else { + return nil + } +} + +func (i *ParenthesesData) SetExpr(value Expression) { + i.expr = value +} + +type ParenthesesImpl struct { + core.AstNodeBase + PrimaryData + ExpressionData + ParenthesesData +} + +func (i *ParenthesesImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.PrimaryData.ForEachNode(fn) + i.ExpressionData.ForEachNode(fn) + i.ParenthesesData.ForEachNode(fn) +} + +func (i *ParenthesesImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.PrimaryData.ForEachReference(fn) + i.ExpressionData.ForEachReference(fn) + i.ParenthesesData.ForEachReference(fn) +} + +func (i *ParenthesesImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameExpr: + if i.Expr() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("ParenthesesImpl.Resolve: field 'expr' is nil in node '%s'", nodePath) + } + child := i.Expr() + return child.Resolve(path.Tail()) + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("ParenthesesImpl.Resolve: field '%s' does not exist in node '%s' of type 'Parentheses'", field.Value(), nodePath) + } +} + +type VariableRef interface { + core.AstNode + Primary + + IsVariableRef() + Name() *core.Reference[VariableDecl] + SetName(value *core.Reference[VariableDecl]) +} + +func NewVariableRef() VariableRef { + return &VariableRefImpl{} +} + +type VariableRefData struct { + name *core.Reference[VariableDecl] +} + +func (i *VariableRefData) IsVariableRef() {} + +func (i *VariableRefData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *VariableRefData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + if i.name != nil { + fn(i.name, fieldNameName, -1) + } +} + +func (i *VariableRefData) Name() *core.Reference[VariableDecl] { + if i != nil && i.name != nil { + return i.name + } else { + return nil + } +} + +func (i *VariableRefData) SetName(value *core.Reference[VariableDecl]) { + i.name = value +} + +type VariableRefImpl struct { + core.AstNodeBase + PrimaryData + ExpressionData + VariableRefData +} + +func (i *VariableRefImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.PrimaryData.ForEachNode(fn) + i.ExpressionData.ForEachNode(fn) + i.VariableRefData.ForEachNode(fn) +} + +func (i *VariableRefImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.PrimaryData.ForEachReference(fn) + i.ExpressionData.ForEachReference(fn) + i.VariableRefData.ForEachReference(fn) +} + +func (i *VariableRefImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameName: + return nil, fmt.Errorf("VariableRefImpl.Resolve: field 'name' is a cross-reference instead of a container field") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("VariableRefImpl.Resolve: field '%s' does not exist in node '%s' of type 'VariableRef'", field.Value(), nodePath) + } +} + +type NumericLiteral interface { + core.AstNode + Primary + + IsNumericLiteral() + Value() string + ValueToken() *core.Token + SetValue(value *core.Token) +} + +func NewNumericLiteral() NumericLiteral { + return &NumericLiteralImpl{} +} + +type NumericLiteralData struct { + value *core.Token +} + +func (i *NumericLiteralData) IsNumericLiteral() {} + +func (i *NumericLiteralData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *NumericLiteralData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *NumericLiteralData) Value() string { + if i != nil && i.value != nil { + return i.value.Image + } else { + return "" + } +} + +func (i *NumericLiteralData) ValueToken() *core.Token { + return i.value +} + +func (i *NumericLiteralData) SetValue(value *core.Token) { + i.value = value +} + +type NumericLiteralImpl struct { + core.AstNodeBase + PrimaryData + ExpressionData + NumericLiteralData +} + +func (i *NumericLiteralImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.PrimaryData.ForEachNode(fn) + i.ExpressionData.ForEachNode(fn) + i.NumericLiteralData.ForEachNode(fn) +} + +func (i *NumericLiteralImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.PrimaryData.ForEachReference(fn) + i.ExpressionData.ForEachReference(fn) + i.NumericLiteralData.ForEachReference(fn) +} + +func (i *NumericLiteralImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameValue: + return nil, fmt.Errorf("NumericLiteralImpl.Resolve: field 'value' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("NumericLiteralImpl.Resolve: field '%s' does not exist in node '%s' of type 'NumericLiteral'", field.Value(), nodePath) + } +} + +type StringLiteral interface { + core.AstNode + Primary + + IsStringLiteral() + Content() []StringContent + SetContentItem(item StringContent) +} + +func NewStringLiteral() StringLiteral { + return &StringLiteralImpl{} +} + +type StringLiteralData struct { + content []StringContent +} + +func (i *StringLiteralData) IsStringLiteral() {} + +func (i *StringLiteralData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + for j, item := range i.content { + fn(item, fieldNameContent, j) + } +} + +func (i *StringLiteralData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *StringLiteralData) Content() []StringContent { + return i.content +} + +func (i *StringLiteralData) SetContentItem(item StringContent) { + i.content = append(i.content, item) +} + +type StringLiteralImpl struct { + core.AstNodeBase + PrimaryData + ExpressionData + StringLiteralData +} + +func (i *StringLiteralImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.PrimaryData.ForEachNode(fn) + i.ExpressionData.ForEachNode(fn) + i.StringLiteralData.ForEachNode(fn) +} + +func (i *StringLiteralImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.PrimaryData.ForEachReference(fn) + i.ExpressionData.ForEachReference(fn) + i.StringLiteralData.ForEachReference(fn) +} + +func (i *StringLiteralImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, index := path.Head() + switch field { + case fieldNameContent: + if index >= len(i.Content()) { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("StringLiteralImpl.Resolve: index %d exceeds length of slice in 'content' (length=%d) in node '%s'", index, len(i.Content()), nodePath) + } + child := i.Content()[index] + if child == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("StringLiteralImpl.Resolve: item %d of slice in field 'content' is nil in node '%s'", index, nodePath) + } + return child.Resolve(path.Tail()) + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("StringLiteralImpl.Resolve: field '%s' does not exist in node '%s' of type 'StringLiteral'", field.Value(), nodePath) + } +} + +type StringContent interface { + core.AstNode + + IsStringContent() +} + +func NewStringContent() StringContent { + return &StringContentImpl{} +} + +type StringContentData struct { +} + +func (i *StringContentData) IsStringContent() {} + +func (i *StringContentData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *StringContentData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +type StringContentImpl struct { + core.AstNodeBase + StringContentData +} + +func (i *StringContentImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.StringContentData.ForEachNode(fn) +} + +func (i *StringContentImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.StringContentData.ForEachReference(fn) +} + +func (i *StringContentImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("StringContentImpl.Resolve: field '%s' does not exist in node '%s' of type 'StringContent'", field.Value(), nodePath) +} + +type StringText interface { + core.AstNode + StringContent + + IsStringText() + Value() string + ValueToken() *core.Token + SetValue(value *core.Token) +} + +func NewStringText() StringText { + return &StringTextImpl{} +} + +type StringTextData struct { + value *core.Token +} + +func (i *StringTextData) IsStringText() {} + +func (i *StringTextData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *StringTextData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *StringTextData) Value() string { + if i != nil && i.value != nil { + return i.value.Image + } else { + return "" + } +} + +func (i *StringTextData) ValueToken() *core.Token { + return i.value +} + +func (i *StringTextData) SetValue(value *core.Token) { + i.value = value +} + +type StringTextImpl struct { + core.AstNodeBase + StringContentData + StringTextData +} + +func (i *StringTextImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.StringContentData.ForEachNode(fn) + i.StringTextData.ForEachNode(fn) +} + +func (i *StringTextImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.StringContentData.ForEachReference(fn) + i.StringTextData.ForEachReference(fn) +} + +func (i *StringTextImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameValue: + return nil, fmt.Errorf("StringTextImpl.Resolve: field 'value' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("StringTextImpl.Resolve: field '%s' does not exist in node '%s' of type 'StringText'", field.Value(), nodePath) + } +} + +type Interpolation interface { + core.AstNode + StringContent + + IsInterpolation() + Expression() Expression + SetExpression(value Expression) +} + +func NewInterpolation() Interpolation { + return &InterpolationImpl{} +} + +type InterpolationData struct { + expression Expression +} + +func (i *InterpolationData) IsInterpolation() {} + +func (i *InterpolationData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + if i.expression != nil { + fn(i.expression, fieldNameExpression, -1) + } +} + +func (i *InterpolationData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *InterpolationData) Expression() Expression { + if i != nil && i.expression != nil { + return i.expression + } else { + return nil + } +} + +func (i *InterpolationData) SetExpression(value Expression) { + i.expression = value +} + +type InterpolationImpl struct { + core.AstNodeBase + StringContentData + InterpolationData +} + +func (i *InterpolationImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.StringContentData.ForEachNode(fn) + i.InterpolationData.ForEachNode(fn) +} + +func (i *InterpolationImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.StringContentData.ForEachReference(fn) + i.InterpolationData.ForEachReference(fn) +} + +func (i *InterpolationImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameExpression: + if i.Expression() == nil { + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("InterpolationImpl.Resolve: field 'expression' is nil in node '%s'", nodePath) + } + child := i.Expression() + return child.Resolve(path.Tail()) + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("InterpolationImpl.Resolve: field '%s' does not exist in node '%s' of type 'Interpolation'", field.Value(), nodePath) + } +} + +var ( + fieldNameContent = unique.Make("content") + fieldNameExpr = unique.Make("expr") + fieldNameExpression = unique.Make("expression") + fieldNameLeft = unique.Make("left") + fieldNameName = unique.Make("name") + fieldNameOperator = unique.Make("operator") + fieldNameRight = unique.Make("right") + fieldNameStatements = unique.Make("statements") + fieldNameValue = unique.Make("value") +) + +var TokenModesSyntheticFactories = map[string]func() core.AstNode{ + "BinaryExpression": func() core.AstNode { return NewBinaryExpression() }, + "Expression": func() core.AstNode { return NewExpression() }, + "Interpolation": func() core.AstNode { return NewInterpolation() }, + "Model": func() core.AstNode { return NewModel() }, + "NumericLiteral": func() core.AstNode { return NewNumericLiteral() }, + "Parentheses": func() core.AstNode { return NewParentheses() }, + "Primary": func() core.AstNode { return NewPrimary() }, + "Statement": func() core.AstNode { return NewStatement() }, + "StringContent": func() core.AstNode { return NewStringContent() }, + "StringLiteral": func() core.AstNode { return NewStringLiteral() }, + "StringText": func() core.AstNode { return NewStringText() }, + "VariableDecl": func() core.AstNode { return NewVariableDecl() }, + "VariableRef": func() core.AstNode { return NewVariableRef() }, +} diff --git a/internal/languages/token_modes_with_groups/atn.md b/internal/languages/token_modes_with_groups/atn.md new file mode 100644 index 00000000..2effe5b6 --- /dev/null +++ b/internal/languages/token_modes_with_groups/atn.md @@ -0,0 +1,18 @@ +# Runtime ATN for token_modes_with_groups + +## Model + +```mermaid +flowchart TD + q0(["Model__Start (0)
RuleStart"]) + q1(["Model__Stop (1)
RuleStop"]) + q2["Model_x (2)
Basic
"] + q3["Model__Basic_0 (3)
Basic
"] + q4["Model__Basic_1 (4)
Basic
"] + + q0 --> q2 + q2 -->|"tok(Keyword_x)"| q3 + q3 -->|"tok(TokenGroup_X)"| q4 + q4 --> q1 +``` + diff --git a/internal/languages/token_modes_with_groups/atn_gen.go b/internal/languages/token_modes_with_groups/atn_gen.go new file mode 100644 index 00000000..57e03618 --- /dev/null +++ b/internal/languages/token_modes_with_groups/atn_gen.go @@ -0,0 +1,49 @@ +// Code generated by fastbelt. DO NOT EDIT. + +package token_modes_with_groups + +import ( + "sync" + "typefox.dev/fastbelt/parser" +) + +const ( + Model__Start = iota + Model__Stop + Model_x + Model__Basic_0 + Model__Basic_1 +) + +var once sync.Once +var atn *parser.RuntimeATN + +func ATN() *parser.RuntimeATN { + once.Do(func() { + atn = BuildATN() + }) + return atn +} +func BuildATN() *parser.RuntimeATN { + states := make([]*parser.RuntimeATNState, 5) + states[Model__Start] = parser.NewATNState(Model__Start, parser.ATNRuleStart, true) + states[Model__Stop] = parser.NewATNState(Model__Stop, parser.ATNRuleStop, false) + states[Model_x] = parser.NewATNState(Model_x, parser.ATNBasic, false) + states[Model__Basic_0] = parser.NewATNState(Model__Basic_0, parser.ATNBasic, false) + states[Model__Basic_1] = parser.NewATNState(Model__Basic_1, parser.ATNBasic, true) + states[Model__Start].AppendTransitions( + parser.NewEpsilonTransition(states[Model_x]), + ) + states[Model_x].AppendTransitions( + parser.NewAtomTransition(states[Model__Basic_0], Keyword_x, nil), + ) + states[Model__Basic_0].AppendTransitions( + parser.NewAtomTransition(states[Model__Basic_1], TokenGroup_X, nil), + ) + states[Model__Basic_1].AppendTransitions( + parser.NewEpsilonTransition(states[Model__Stop]), + ) + decisionStates := make([]*parser.RuntimeATNState, 0) + decisionMap := make([]*parser.RuntimeATNState, 0) + return parser.NewRuntimeATN(states, decisionStates, decisionMap) +} diff --git a/internal/languages/token_modes_with_groups/completion_gen.go b/internal/languages/token_modes_with_groups/completion_gen.go new file mode 100644 index 00000000..d6faa068 --- /dev/null +++ b/internal/languages/token_modes_with_groups/completion_gen.go @@ -0,0 +1,71 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes_with_groups + +import ( + "context" + "iter" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/parser" + "typefox.dev/fastbelt/util/service" +) + +// TokenModesWithGroupsCompletionFilter lets adopters refine the candidates produced by +// the existing TokenModesWithGroupsScopeProvider when completing a cross-reference. +type TokenModesWithGroupsCompletionFilter interface { +} + +type DefaultTokenModesWithGroupsCompletionFilter struct{} + +func NewDefaultTokenModesWithGroupsCompletionFilter() TokenModesWithGroupsCompletionFilter { + return &DefaultTokenModesWithGroupsCompletionFilter{} +} + +var TokenModesWithGroupsCompletionDispatch = map[string]TokenModesWithGroupsCompletionDispatchFunc{} + +type TokenModesWithGroupsCompletionDispatchFunc func(ctx context.Context, sc *core.AstNode) iter.Seq[*core.SymbolDescription] + +type TokenModesWithGroupsCompletionAdapter struct { + sc *service.Container +} + +func NewTokenModesWithGroupsCompletionAdapter(sc *service.Container) *TokenModesWithGroupsCompletionAdapter { + return &TokenModesWithGroupsCompletionAdapter{sc: sc} +} + +func (a *TokenModesWithGroupsCompletionAdapter) Parse(tokens []core.Token) *parser.CompletionParseResult { + return NewCompletionParser(a.sc).Parse(tokens) +} + +func (a *TokenModesWithGroupsCompletionAdapter) ATN() *parser.RuntimeATN { + return BuildATN() +} + +func (a *TokenModesWithGroupsCompletionAdapter) SyntheticOwnerFor(ruleKey string) (core.AstNode, bool) { + factory, ok := TokenModesWithGroupsSyntheticFactories[ruleKey] + if !ok { + return nil, false + } + return factory(), true +} + +func (a *TokenModesWithGroupsCompletionAdapter) DispatchCompletion(ctx context.Context, field string, owner core.AstNode) (iter.Seq[*core.SymbolDescription], bool) { + _ = field + _ = owner + _ = ctx + return nil, false +} + +func (a *TokenModesWithGroupsCompletionAdapter) HasAssignment(node core.AstNode, property string) bool { + _ = node + _ = property + return false +} + +func (a *TokenModesWithGroupsCompletionAdapter) ApplyAction(actionType, property string, value core.AstNode) core.AstNode { + _ = actionType + _ = property + _ = value + return nil +} diff --git a/internal/languages/token_modes_with_groups/completion_parser_gen.go b/internal/languages/token_modes_with_groups/completion_parser_gen.go new file mode 100644 index 00000000..90289f73 --- /dev/null +++ b/internal/languages/token_modes_with_groups/completion_parser_gen.go @@ -0,0 +1,53 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes_with_groups + +import ( + "sync" + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/parser" + "typefox.dev/fastbelt/util/service" +) + +type CompletionParser struct { + state *parser.ParserState + cp *parser.CompletionParserState + sc *service.Container + atn func() *parser.RuntimeATN + lookahead TokenModesWithGroupsParserLookahead +} + +func NewCompletionParser(sc *service.Container) *CompletionParser { + return &CompletionParser{ + sc: sc, + atn: sync.OnceValue(BuildATN), + } +} + +// Parse runs the completion parser over the given prefix tokens (typically +// the document's tokens up to the cursor) and returns the recorded +// snapshots and rule stack. The completion provider feeds that result into +// the ATN simulator. +func (p *CompletionParser) Parse(tokens []core.Token) *parser.CompletionParseResult { + messages := service.MustGet[parser.ErrorMessageProvider](p.sc) + recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) + lookahead := service.MustGet[TokenModesWithGroupsParserLookahead](p.sc) + cp := &CompletionParser{sc: p.sc, atn: p.atn, lookahead: lookahead} + cp.state = parser.NewParserState(tokens, p.atn(), recovery, messages) + cp.cp = parser.NewCompletionParserState(cp.state) + cp.ParseModel() + return cp.cp.Result(tokens) +} + +func (p *CompletionParser) ParseModel() { + p.cp.EnterRule("Model", Model__Start) + defer p.cp.ExitRule() + { + p.state.Consume(Keyword_x) + } + { + p.cp.MarkAssignment("Greeting") + p.state.Consume(TokenGroup_X) + p.cp.ClearAssignment() + } +} diff --git a/internal/languages/token_modes_with_groups/lexer_gen.go b/internal/languages/token_modes_with_groups/lexer_gen.go new file mode 100644 index 00000000..be7e2218 --- /dev/null +++ b/internal/languages/token_modes_with_groups/lexer_gen.go @@ -0,0 +1,52 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes_with_groups + +import ( + "strings" + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/lexer" +) + +const Keyword_x_Idx = 1 + +var Keyword_x = core.NewTokenType( + Keyword_x_Idx, + "x", + "x", + core.TokenKindKeyword, + func(text string, offset int) int { + if strings.HasPrefix(text[offset:], "x") { + return 1 + } + return 0 + }, + []rune{'x'}, +) + +const TokenGroup_X_Idx = 2 + +var TokenGroup_X = core.NewTokenGroup( + TokenGroup_X_Idx, + "X", + "X", + []*core.TokenType{ + Keyword_x, + }, +) + +const ( + TokenMode_default = 0 + TokenMode_Y = 1 +) + +func NewLexer() lexer.Lexer { + modes := make([]*lexer.TokenMode, 2) + modes[TokenMode_default] = lexer.NewTokenMode("default", + lexer.UseTokenType(Keyword_x).WithPushMode(TokenMode_Y), + ) + modes[TokenMode_Y] = lexer.NewTokenMode("Y", + lexer.UseTokenType(Keyword_x).WithPopMode(), + ) + return lexer.NewDefaultLexer(TokenMode_default, modes...) +} diff --git a/internal/languages/token_modes_with_groups/linker_gen.go b/internal/languages/token_modes_with_groups/linker_gen.go new file mode 100644 index 00000000..ff298935 --- /dev/null +++ b/internal/languages/token_modes_with_groups/linker_gen.go @@ -0,0 +1,81 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes_with_groups + +import ( + "reflect" + "sync" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/util/service" +) + +type TokenModesWithGroupsScopeProvider interface { +} + +type DefaultTokenModesWithGroupsScopeProvider struct { + sc *service.Container +} + +func NewDefaultTokenModesWithGroupsScopeProvider(sc *service.Container) TokenModesWithGroupsScopeProvider { + return &DefaultTokenModesWithGroupsScopeProvider{sc: sc} +} + +type TokenModesWithGroupsReferenceLinker interface { +} + +type DefaultTokenModesWithGroupsReferenceLinker struct { + sc *service.Container + scopeProvider func() TokenModesWithGroupsScopeProvider +} + +func NewDefaultTokenModesWithGroupsReferenceLinker(sc *service.Container) TokenModesWithGroupsReferenceLinker { + return &DefaultTokenModesWithGroupsReferenceLinker{ + sc: sc, + scopeProvider: sync.OnceValue(func() TokenModesWithGroupsScopeProvider { + return service.MustGet[TokenModesWithGroupsScopeProvider](sc) + }), + } +} + +type TokenModesWithGroupsReferencesConstructor interface { +} + +type DefaultTokenModesWithGroupsReferencesConstructor struct { + sc *service.Container + referenceLinker func() TokenModesWithGroupsReferenceLinker +} + +func NewDefaultTokenModesWithGroupsReferencesConstructor(sc *service.Container) TokenModesWithGroupsReferencesConstructor { + return &DefaultTokenModesWithGroupsReferencesConstructor{ + sc: sc, + referenceLinker: sync.OnceValue(func() TokenModesWithGroupsReferenceLinker { + return service.MustGet[TokenModesWithGroupsReferenceLinker](sc) + }), + } +} + +type TokenModesWithGroupsSymbolContainers struct{} + +func (c *TokenModesWithGroupsSymbolContainers) New() core.SymbolContainer { + return &TokenModesWithGroupsSymbolContainer{} +} + +func NewSymbolContainers() *TokenModesWithGroupsSymbolContainers { + return &TokenModesWithGroupsSymbolContainers{} +} + +type TokenModesWithGroupsSymbolContainer struct { +} + +func (sc *TokenModesWithGroupsSymbolContainer) Put(desc *core.SymbolDescription) bool { + return false +} + +func (sc *TokenModesWithGroupsSymbolContainer) All() core.SymbolSeq { + return core.EmptySymbolDescriptions +} + +func (sc *TokenModesWithGroupsSymbolContainer) ForType(t reflect.Type) core.SymbolSeq { + return core.EmptySymbolDescriptions +} diff --git a/internal/languages/token_modes_with_groups/parser_gen.go b/internal/languages/token_modes_with_groups/parser_gen.go new file mode 100644 index 00000000..980d8816 --- /dev/null +++ b/internal/languages/token_modes_with_groups/parser_gen.go @@ -0,0 +1,53 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes_with_groups + +import ( + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/parser" + "typefox.dev/fastbelt/util/service" +) + +type Parser struct { + state *parser.ParserState + sc *service.Container + referencesConstructor TokenModesWithGroupsReferencesConstructor + lookahead TokenModesWithGroupsParserLookahead +} + +func (p *Parser) Parse(document *core.Document) *parser.ParseResult { + recovery := service.MustGet[parser.ErrorRecoveryStrategy](p.sc) + messages := service.MustGet[parser.ErrorMessageProvider](p.sc) + referencesConstructor := service.MustGet[TokenModesWithGroupsReferencesConstructor](p.sc) + lookahead := service.MustGet[TokenModesWithGroupsParserLookahead](p.sc) + cp := &Parser{sc: p.sc, referencesConstructor: referencesConstructor, lookahead: lookahead, state: parser.NewParserState(document.Tokens, ATN(), recovery, messages)} + result := cp.ParseModel() + cp.state.ExpectEndOfInput() + return &parser.ParseResult{Node: result, Errors: cp.state.Errors()} +} + +func NewParser(sc *service.Container) *Parser { + return &Parser{ + sc: sc, + } +} + +func (p *Parser) ParseModel() Model { + current := NewModel() + current.SetTextRangeStart(p.state.LA(1).Range.Start) + { + { + token := p.state.Consume(Keyword_x) + core.AssignToken(current, token, Model_x) + } + { + token := p.state.Consume(TokenGroup_X) + core.AssignToken(current, token, Model__Basic_0) + if token != nil { + current.SetGreeting(token) + } + } + } + current.SetTextRangeEnd(p.state.LA(0).Range.End) + return current +} diff --git a/internal/languages/token_modes_with_groups/parser_lookahead_gen.go b/internal/languages/token_modes_with_groups/parser_lookahead_gen.go new file mode 100644 index 00000000..fb40b232 --- /dev/null +++ b/internal/languages/token_modes_with_groups/parser_lookahead_gen.go @@ -0,0 +1,25 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes_with_groups + +import ( + "typefox.dev/fastbelt/parser" +) + +// TokenModesWithGroupsParserLookahead abstracts every lookahead/prediction decision performed by +// the generated parser. Each method corresponds to a single decision point; +// implementations may override individual decisions while delegating the rest +// to DefaultTokenModesWithGroupsParserLookahead. +type TokenModesWithGroupsParserLookahead interface { + parser.ParserLookahead +} + +// DefaultTokenModesWithGroupsParserLookahead resolves every decision with the parser state's built-in +// static LL(1) and adaptive ALL(*) prediction. +type DefaultTokenModesWithGroupsParserLookahead struct { + parser.DefaultParserLookahead +} + +func NewDefaultTokenModesWithGroupsParserLookahead() TokenModesWithGroupsParserLookahead { + return &DefaultTokenModesWithGroupsParserLookahead{} +} diff --git a/internal/languages/token_modes_with_groups/parser_test.go b/internal/languages/token_modes_with_groups/parser_test.go new file mode 100644 index 00000000..c9604402 --- /dev/null +++ b/internal/languages/token_modes_with_groups/parser_test.go @@ -0,0 +1,18 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package token_modes_with_groups + +import ( + "testing" + + "github.com/stretchr/testify/require" + "typefox.dev/fastbelt/test" +) + +func TestXX(t *testing.T) { + fixture := test.New(t, CreateServices()) + doc := fixture.Parse("xx") + require.Len(t, doc.Document.Tokens, 2) +} diff --git a/internal/languages/token_modes_with_groups/services.go b/internal/languages/token_modes_with_groups/services.go new file mode 100644 index 00000000..f43867ee --- /dev/null +++ b/internal/languages/token_modes_with_groups/services.go @@ -0,0 +1,33 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package token_modes_with_groups + +//go:generate go run ../../../cmd/fastbelt generate ./token_modes_with_groups.fb -v + +import ( + "typefox.dev/fastbelt/linking" + "typefox.dev/fastbelt/server" + "typefox.dev/fastbelt/textdoc" + "typefox.dev/fastbelt/util/service" + "typefox.dev/fastbelt/workspace" +) + +func SetupServices(sc *service.Container) { + service.Put[workspace.LanguageID](sc, "modes_with_groups") + service.Put[workspace.FileExtensions](sc, []string{".mode_with_groups"}) + textdoc.SetupDefaultServices(sc) + linking.SetupDefaultServices(sc) + workspace.SetupDefaultServices(sc) + SetupGeneratedServices(sc) + SetupGeneratedServerServices(sc) + server.SetupDefaultServices(sc) +} + +func CreateServices() *service.Container { + sc := service.NewContainer() + SetupServices(sc) + sc.Seal() + return sc +} diff --git a/internal/languages/token_modes_with_groups/services_gen.go b/internal/languages/token_modes_with_groups/services_gen.go new file mode 100644 index 00000000..0429fae9 --- /dev/null +++ b/internal/languages/token_modes_with_groups/services_gen.go @@ -0,0 +1,53 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes_with_groups + +import ( + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/lexer" + "typefox.dev/fastbelt/parser" + "typefox.dev/fastbelt/util/service" +) + +// SetupGeneratedServices sets up the generated services for this grammar. +// If any service is already set, it's not overwritten. +func SetupGeneratedServices(sc *service.Container) { + if !service.Has[TokenModesWithGroupsScopeProvider](sc) { + service.Put(sc, NewDefaultTokenModesWithGroupsScopeProvider(sc)) + } + if !service.Has[TokenModesWithGroupsReferenceLinker](sc) { + service.Put(sc, NewDefaultTokenModesWithGroupsReferenceLinker(sc)) + } + if !service.Has[TokenModesWithGroupsReferencesConstructor](sc) { + service.Put(sc, NewDefaultTokenModesWithGroupsReferencesConstructor(sc)) + } + if !service.Has[TokenModesWithGroupsParserLookahead](sc) { + service.Put(sc, NewDefaultTokenModesWithGroupsParserLookahead()) + } + if !service.Has[lexer.Lexer](sc) { + service.Put(sc, NewLexer()) + } + if !service.Has[parser.Parser](sc) { + service.Put[parser.Parser](sc, NewParser(sc)) + } + if !service.Has[parser.ErrorRecoveryStrategy](sc) { + service.Put[parser.ErrorRecoveryStrategy](sc, parser.NewDefaultErrorRecovery()) + } + if !service.Has[parser.ErrorMessageProvider](sc) { + service.Put[parser.ErrorMessageProvider](sc, parser.NewDefaultErrorMessageProvider()) + } + if !service.Has[core.SymbolContainers](sc) { + service.Put[core.SymbolContainers](sc, NewSymbolContainers()) + } +} + +// SetupGeneratedServerServices sets up the generated language server services for this grammar. +// If any service is already set, it's not overwritten. +func SetupGeneratedServerServices(sc *service.Container) { + if !service.Has[TokenModesWithGroupsCompletionFilter](sc) { + service.Put(sc, NewDefaultTokenModesWithGroupsCompletionFilter()) + } + if !service.Has[parser.LanguageCompletionAdapter](sc) { + service.Put[parser.LanguageCompletionAdapter](sc, NewTokenModesWithGroupsCompletionAdapter(sc)) + } +} diff --git a/internal/languages/token_modes_with_groups/token_modes_with_groups.fb b/internal/languages/token_modes_with_groups/token_modes_with_groups.fb new file mode 100644 index 00000000..eaf72e43 --- /dev/null +++ b/internal/languages/token_modes_with_groups/token_modes_with_groups.fb @@ -0,0 +1,18 @@ +grammar TokenModesWithGroups + +interface Model { + Greeting string +} +entry Model: "x" Greeting=X + +token group X { + "x" +} + +token mode default { + "x" -> push(Y) +} + +token mode Y { + X -> pop +} \ No newline at end of file diff --git a/internal/languages/token_modes_with_groups/types_gen.go b/internal/languages/token_modes_with_groups/types_gen.go new file mode 100644 index 00000000..648eed09 --- /dev/null +++ b/internal/languages/token_modes_with_groups/types_gen.go @@ -0,0 +1,86 @@ +// Code generated by typefox.dev/fastbelt/cmd/fastbelt. DO NOT EDIT. + +package token_modes_with_groups + +import ( + "fmt" + "unique" + + core "typefox.dev/fastbelt" +) + +type Model interface { + core.AstNode + + IsModel() + Greeting() string + GreetingToken() *core.Token + SetGreeting(value *core.Token) +} + +func NewModel() Model { + return &ModelImpl{} +} + +type ModelData struct { + greeting *core.Token +} + +func (i *ModelData) IsModel() {} + +func (i *ModelData) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { +} + +func (i *ModelData) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { +} + +func (i *ModelData) Greeting() string { + if i != nil && i.greeting != nil { + return i.greeting.Image + } else { + return "" + } +} + +func (i *ModelData) GreetingToken() *core.Token { + return i.greeting +} + +func (i *ModelData) SetGreeting(value *core.Token) { + i.greeting = value +} + +type ModelImpl struct { + core.AstNodeBase + ModelData +} + +func (i *ModelImpl) ForEachNode(fn func(core.AstNode, unique.Handle[string], int)) { + i.ModelData.ForEachNode(fn) +} + +func (i *ModelImpl) ForEachReference(fn func(core.UntypedReference, unique.Handle[string], int)) { + i.ModelData.ForEachReference(fn) +} + +func (i *ModelImpl) Resolve(path core.FragmentPath) (core.AstNode, error) { + if path.Empty() { + return i, nil + } + field, _ := path.Head() + switch field { + case fieldNameGreeting: + return nil, fmt.Errorf("ModelImpl.Resolve: field 'greeting' holds a primitive value instead of an ast node") + default: + nodePath, _ := core.PathOf(i) + return nil, fmt.Errorf("ModelImpl.Resolve: field '%s' does not exist in node '%s' of type 'Model'", field.Value(), nodePath) + } +} + +var ( + fieldNameGreeting = unique.Make("greeting") +) + +var TokenModesWithGroupsSyntheticFactories = map[string]func() core.AstNode{ + "Model": func() core.AstNode { return NewModel() }, +} diff --git a/internal/vscode-extensions/fastbelt/data/fastbelt.tmLanguage.json b/internal/vscode-extensions/fastbelt/data/fastbelt.tmLanguage.json index 9e88e818..f92c86d8 100644 --- a/internal/vscode-extensions/fastbelt/data/fastbelt.tmLanguage.json +++ b/internal/vscode-extensions/fastbelt/data/fastbelt.tmLanguage.json @@ -13,7 +13,7 @@ }, { "name": "keyword.control.fastbelt", - "match": "\\b(bool|comment|composite|current|entry|extends|fragment|grammar|group|hidden|import|interface|keywords|returns|string|token|type)\\b" + "match": "\\b(bool|comment|composite|current|entry|default|extends|fragment|grammar|group|hidden|import|interface|keywords|mode|pop|push|returns|string|token|type)\\b" }, { "name": "constant.language.fastbelt", diff --git a/lexer/doc.go b/lexer/doc.go index a36886b8..75495d8a 100644 --- a/lexer/doc.go +++ b/lexer/doc.go @@ -18,39 +18,56 @@ // Code generation turns grammar terminals into Go values: // // - Keyword literals from parser rules become [core.TokenType] values with -// prefix [core.Matcher] functions and [core.TokenKindKeyword]. +// prefix [core.TokenMatcher] functions and [core.TokenKindKeyword]. // - Named `token` rules become [core.TokenType] values whose matchers are -// compiled from the rule's regular expression, with [core.StartChars] taken -// from the expression's possible first runes (the StartChars field). +// compiled from the rule's regular expression, with StartChars taken from +// the expression's possible first runes. // // The generated `NewLexer` function returns a [DefaultLexer] constructed via -// [NewDefaultLexer] with every keyword and token type for that language. -// [typefox.dev/fastbelt/workspace.DefaultDocumentParser] obtains a [Lexer] from -// the service container, calls [Lexer.Lex], and stores [LexerResult.Tokens], -// [LexerResult.Comments], and [LexerResult.Errors] on the document before -// parsing. +// [NewDefaultLexer] with one [TokenMode] per `token mode` declaration in the +// grammar. [typefox.dev/fastbelt/workspace.DefaultDocumentParser] obtains a +// [Lexer] from the service container, calls [Lexer.Exec], and stores +// [LexerResult.Tokens], [LexerResult.Comments], and [LexerResult.Errors] on the +// document before parsing. // // # Lexing model // // Fastbelt uses a table-driven scanner with the following approach: // // 1. At each input offset, decode the current rune and look up candidate token -// types in a fixed-size map keyed by rune % 256 (each [core.TokenType]'s -// StartChars slice). -// 2. Run each candidate's [core.Matcher] and keep the longest match (maximal -// munch). Among equal-length matches, the first registered token type wins; -// generated lexers list keywords before regex token rules, so keywords take -// precedence when both match the same span. -// 3. Route the match by [core.TokenType.Group]: default tokens go to +// types in the active mode's fixed-size map keyed by rune % 256 (built from +// each [core.TokenType]'s StartChars slice). +// 2. Run each candidate's [core.TokenMatcher] and keep the longest match +// (maximal munch). Among equal-length matches, the first registered token +// type wins; generated lexers list keywords before regex token rules, so +// keywords take precedence when both match the same span. +// 3. Route the match by [TokenTypeUsage.Modifier]: default tokens go to // [LexerResult.Tokens], hidden tokens are dropped, comments go to -// [LexerResult.Comments], and other groups are collected in -// [LexerResult.Groups]. -// 4. If no token type matches, emit a [core.LexerError] and advance by one -// UTF-8 code point so lexing can continue. -// -// [DefaultLexer] tracks line and column while scanning and adapts its initial -// token-slice capacity from a running average of tokens per byte across prior -// [Lexer.Lex] calls on the same instance. +// [LexerResult.Comments], and other modifiers are collected in +// [LexerResult.Modifiers]. +// 4. Apply the match's mode command, if any (see below). +// 5. If no token type matches, emit a [core.LexerError] and advance by one +// UTF-8 code point so lexing can continue. The active mode is unchanged. +// +// [DefaultLexer] adapts its initial token-slice capacity from a running average +// of tokens per byte across prior [Lexer.Exec] calls on the same instance. +// +// # Token modes +// +// A [TokenMode] is a named set of token types. Only the active mode's tokens are +// candidates at a given offset, which is how a language can lex, say, string +// interpolation (aka string templates) or nested comments. Each [Lexer.Exec] starts +// in the mode passed to [NewDefaultLexer] and tracks the active one in its own +// [TokenModeStack], so a single [DefaultLexer] is safe for concurrent use and +// input that ends inside a pushed mode cannot affect the next call. +// +// A token type registered with a mode may carry one command: +// +// - push: make another mode active, remembering the current one. +// - pop: return to the mode remembered by the matching push. A pop with +// nothing to return to keeps the start mode active rather than failing. +// - mode: replace the active mode without remembering it, so a later pop +// returns to whatever was below it rather than to the replaced mode. // // Custom language projects rarely import this package directly unless they // replace the generated lexer. Typical integration is to call the generated diff --git a/lexer/lexer.go b/lexer/lexer.go index 45f7ea74..56f8b860 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -13,21 +13,21 @@ import ( // Lexer tokenizes a complete source string in one shot. type Lexer interface { - Lex(input string) *LexerResult + Exec(input string) *LexerResult } -// LexerResult holds everything produced by a single [Lexer.Lex] pass over +// LexerResult holds everything produced by a single [Lexer.Exec] pass over // source text. type LexerResult struct { // Tokens is the main token stream passed to the parser. Tokens []core.Token - // Comments holds tokens whose [core.TokenType] uses [core.CommentGroup]. + // Comments holds tokens whose [core.TokenType] uses [core.CommentModifier]. Comments []core.Token // Errors lists recoverable lexing problems (unrecognized input). Errors []*core.LexerError - // Groups collects tokens routed to custom [core.TokenType.Group] values - // other than the default, skipped, or comment groups. Nil when empty. - Groups map[int][]core.Token + // Modifiers collects tokens routed to custom [TokenTypeUsage.Modifier] values + // other than the default, skipped, or comment modifiers. Nil when empty. + Modifiers map[int][]core.Token } // Allocate a new token every ~5 characters on average @@ -38,33 +38,41 @@ const defaultTokenRatio = 1.0 / 5.0 // functions build one from the [core.TokenType] descriptors emitted for a // grammar. type DefaultLexer struct { - tokenTypes []*core.TokenType - tokenMap [][]*core.TokenType + tokenModes []*TokenMode + // index into tokenModes of the mode every Exec starts in + defaultMode int // running exponential moving average of tokens-per-byte avgRatio *parallel.RunningAverage } -// Lex scans input from left to right using longest-match disambiguation among +// Exec scans input from left to right using longest-match disambiguation among // token types registered at construction time. -func (l *DefaultLexer) Lex(input string) *LexerResult { +func (l *DefaultLexer) Exec(input string) *LexerResult { length := len(input) tokens := make([]core.Token, 0, l.avgRatio.Capacity(length)) comments := make([]core.Token, 0) errors := make([]*core.LexerError, 0) - var groups map[int][]core.Token + var modifiers map[int][]core.Token + + // The mode stack is local to this call: a DefaultLexer is shared between + // documents and Exec may run concurrently, so input that ends inside a + // pushed mode must not leak into the next run. + stack := NewTokenModeStack(l.tokenModes[l.defaultMode]) + currentTokenMode := stack.Peek() var offset int - for offset < length { + for offset < length && currentTokenMode != nil { r, size := utf8.DecodeRuneInString(input[offset:]) mapIndex := int(r) % maxChar - candidates := l.tokenMap[mapIndex] + candidates := currentTokenMode.TokenMap[mapIndex] longestMatch := 0 - var longestType *core.TokenType - for _, tokenType := range candidates { + var longestType *TokenTypeUsage + for _, tokenTypeUsage := range candidates { + tokenType := tokenTypeUsage.TokenType tokenMatch := tokenType.Match(input, offset) if tokenMatch > longestMatch { longestMatch = tokenMatch - longestType = tokenType + longestType = tokenTypeUsage } } @@ -76,31 +84,47 @@ func (l *DefaultLexer) Lex(input string) *LexerResult { end := offset + longestMatch if longestType != nil { - switch longestType.Group { - case core.SkippedGroup: + switch longestType.Modifier { + case core.SkippedModifier: // do nothing - case core.CommentGroup: + case core.CommentModifier: comments = append(comments, core.NewToken( - longestType, + longestType.TokenType, input[offset:end], offset, end, )) case 0: tokens = append(tokens, core.NewToken( - longestType, + longestType.TokenType, input[offset:end], offset, end, )) default: - if groups == nil { - groups = make(map[int][]core.Token) + if modifiers == nil { + modifiers = make(map[int][]core.Token) } - groups[longestType.Group] = append(groups[longestType.Group], core.NewToken( - longestType, + modifiers[longestType.Modifier] = append(modifiers[longestType.Modifier], core.NewToken( + longestType.TokenType, input[offset:end], offset, end, )) } + + switch { + case longestType.PopMode && longestType.PushMode > -1: + // `mode(X)` replaces the active mode without deepening the + // stack, so a later pop returns to whatever was below it rather + // than to the mode that was replaced. This mirrors ANTLR's + // `mode` and is intentional: only `push` can be undone by `pop`. + stack.SetMode(l.tokenModes[longestType.PushMode]) + currentTokenMode = stack.Peek() + case longestType.PopMode: + stack.Pop() + currentTokenMode = stack.Peek() + case longestType.PushMode > -1: + stack.Push(l.tokenModes[longestType.PushMode]) + currentTokenMode = stack.Peek() + } } else { errors = append(errors, core.NewLexerError( "No matching token", @@ -117,33 +141,24 @@ func (l *DefaultLexer) Lex(input string) *LexerResult { } return &LexerResult{ - Tokens: tokens, - Comments: comments, - Errors: errors, - Groups: groups, + Tokens: tokens, + Comments: comments, + Errors: errors, + Modifiers: modifiers, } } const maxChar = 256 -// NewDefaultLexer returns a lexer that recognizes the given token types. -// At each position the longest match wins; among equal-length matches, the -// first argument wins. -func NewDefaultLexer(tokenTypes ...*core.TokenType) *DefaultLexer { - tokenMap := make([][]*core.TokenType, maxChar) - for i := range maxChar { - tokenMap[i] = make([]*core.TokenType, 0) +// NewDefaultLexer returns a [DefaultLexer] that starts every [DefaultLexer.Exec] +// in tokenModes[defaultMode]. The returned lexer is safe for concurrent use. +func NewDefaultLexer(defaultMode int, tokenModes ...*TokenMode) *DefaultLexer { + if defaultMode < 0 || defaultMode >= len(tokenModes) { + panic("lexer: default token mode index out of range") } - for _, tokenType := range tokenTypes { - for _, r := range tokenType.StartChars { - index := int(r) % maxChar - tokenMap[index] = append(tokenMap[index], tokenType) - } - } - return &DefaultLexer{ - tokenTypes: tokenTypes, - tokenMap: tokenMap, - avgRatio: parallel.NewRunningAverage(defaultTokenRatio), + tokenModes: tokenModes, + defaultMode: defaultMode, + avgRatio: parallel.NewRunningAverage(defaultTokenRatio), } } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go new file mode 100644 index 00000000..2acbda20 --- /dev/null +++ b/lexer/lexer_test.go @@ -0,0 +1,394 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package lexer + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + core "typefox.dev/fastbelt" +) + +const ( + lowercase = "abcdefghijklmnopqrstuvwxyz" + spaces = " \t\r\n" +) + +// images returns the token images of tokens, for compact assertions. +func images(tokens []core.Token) []string { + result := make([]string, len(tokens)) + for i, token := range tokens { + result[i] = token.Image + } + return result +} + +// names returns the token type names of tokens. +func names(tokens []core.Token) []string { + result := make([]string, len(tokens)) + for i, token := range tokens { + result[i] = token.Type.Name + } + return result +} + +// stringLexer builds a two-mode lexer: the default mode reads identifiers and +// enters IN_STRING on a quote, IN_STRING reads text and leaves on a quote. +// customGroup is the group assigned to IN_STRING's text token. +func stringLexer(customGroup int) *DefaultLexer { + const ( + modeDefault = 0 + modeString = 1 + ) + id := matchRunes(1, "ID", lowercase) + quote := literal(2, "QUOTE", `"`) + ws := matchRunes(3, "WS", spaces) + text := matchRunes(4, "TEXT", lowercase+spaces) + + modes := make([]*TokenMode, 2) + modes[modeDefault] = NewTokenMode("default", + UseTokenType(quote).WithPushMode(modeString), + UseTokenType(id), + UseTokenType(ws).WithModifier(core.SkippedModifier), + ) + modes[modeString] = NewTokenMode("IN_STRING", + UseTokenType(quote).WithPopMode(), + UseTokenType(text).WithModifier(customGroup), + ) + return NewDefaultLexer(modeDefault, modes...) +} + +// --- Longest match and tie-breaking --- + +func TestExecPrefersLongestMatch(t *testing.T) { + short := literal(1, "SHORT", "a") + long := literal(2, "LONG", "abc") + lexer := NewDefaultLexer(0, NewTokenMode("default", + UseTokenType(short), + UseTokenType(long), + )) + + result := lexer.Exec("abc") + require.Len(t, result.Tokens, 1) + assert.Equal(t, "LONG", result.Tokens[0].Type.Name) +} + +func TestExecFirstRegisteredWinsEqualLengthMatch(t *testing.T) { + first := literal(1, "FIRST", "a") + second := literal(2, "SECOND", "a") + lexer := NewDefaultLexer(0, NewTokenMode("default", + UseTokenType(first), + UseTokenType(second), + )) + + result := lexer.Exec("a") + require.Len(t, result.Tokens, 1) + assert.Equal(t, "FIRST", result.Tokens[0].Type.Name, + "among equal-length matches the first registered token type must win") +} + +// --- Groups --- + +func TestExecRoutesTokensToModifiers(t *testing.T) { + word := matchRunes(1, "WORD", lowercase) + ws := matchRunes(2, "WS", spaces) + comment := literal(3, "COMMENT", "#") + other := literal(4, "OTHER", "!") + lexer := NewDefaultLexer(0, NewTokenMode("default", + UseTokenType(word), + UseTokenType(ws).WithModifier(core.SkippedModifier), + UseTokenType(comment).WithModifier(core.CommentModifier), + UseTokenType(other).WithModifier(7), + )) + + result := lexer.Exec("ab #ff!") + assert.Equal(t, []string{"ab", "ff"}, images(result.Tokens)) + assert.Equal(t, []string{"#"}, images(result.Comments)) + require.Contains(t, result.Modifiers, 7) + assert.Equal(t, []string{"!"}, images(result.Modifiers[7])) + assert.Empty(t, result.Errors) +} + +func TestExecModifiersIsNilWithoutCustomModifiers(t *testing.T) { + word := matchRunes(1, "WORD", lowercase) + lexer := NewDefaultLexer(0, NewTokenMode("default", UseTokenType(word))) + + assert.Nil(t, lexer.Exec("ab").Modifiers) +} + +func TestExecCollectsCustomModifierFromNonDefaultMode(t *testing.T) { + lexer := stringLexer(7) + + result := lexer.Exec(`ab "inside" cd`) + assert.Equal(t, []string{"ab", `"`, `"`, "cd"}, images(result.Tokens)) + require.Contains(t, result.Modifiers, 7) + assert.Equal(t, []string{"inside"}, images(result.Modifiers[7])) +} + +func TestExecSkipsHiddenTokenFromNonDefaultMode(t *testing.T) { + lexer := stringLexer(core.SkippedModifier) + + result := lexer.Exec(`ab "inside" cd`) + assert.Equal(t, []string{"ab", `"`, `"`, "cd"}, images(result.Tokens)) + assert.Nil(t, result.Modifiers) +} + +func TestExecCollectsCommentFromNonDefaultMode(t *testing.T) { + lexer := stringLexer(core.CommentModifier) + + result := lexer.Exec(`ab "inside" cd`) + assert.Equal(t, []string{"inside"}, images(result.Comments)) +} + +// --- Mode switching --- + +func TestExecPushAndPopMode(t *testing.T) { + lexer := stringLexer(0) + + result := lexer.Exec(`ab "in string" cd`) + assert.Equal(t, []string{"ID", "QUOTE", "TEXT", "QUOTE", "ID"}, names(result.Tokens)) + assert.Equal(t, []string{"ab", `"`, "in string", `"`, "cd"}, images(result.Tokens)) + assert.Empty(t, result.Errors) +} + +func TestExecNestedPushMode(t *testing.T) { + // outer -> middle -> inner, then two pops back to outer + outer := literal(1, "OUTER", "o") + enterMiddle := literal(2, "ENTER_MIDDLE", "(") + enterInner := literal(3, "ENTER_INNER", "[") + leave := literal(4, "LEAVE", ")") + middleBody := literal(5, "MIDDLE", "m") + innerBody := literal(6, "INNER", "i") + + modes := make([]*TokenMode, 3) + modes[0] = NewTokenMode("default", + UseTokenType(outer), + UseTokenType(enterMiddle).WithPushMode(1), + ) + modes[1] = NewTokenMode("middle", + UseTokenType(middleBody), + UseTokenType(enterInner).WithPushMode(2), + UseTokenType(leave).WithPopMode(), + ) + modes[2] = NewTokenMode("inner", + UseTokenType(innerBody), + UseTokenType(leave).WithPopMode(), + ) + lexer := NewDefaultLexer(0, modes...) + + result := lexer.Exec("o(m[i))o") + assert.Equal(t, []string{ + "OUTER", "ENTER_MIDDLE", "MIDDLE", "ENTER_INNER", "INNER", "LEAVE", "LEAVE", "OUTER", + }, names(result.Tokens)) + assert.Empty(t, result.Errors) +} + +func TestExecSetModeDoesNotGrowTheModeStack(t *testing.T) { + // `a` switches from the start mode to `second` with a set-mode command. + // Because set-mode replaces the active mode rather than pushing onto the + // stack, the following pop has nothing to return to and `second` stays + // active - which is why `y` from the start mode no longer matches. + a := literal(1, "A", "a") + b := literal(2, "B", "b") + inSecond := literal(3, "IN_SECOND", "x") + inDefault := literal(4, "IN_DEFAULT", "y") + + modes := make([]*TokenMode, 2) + modes[0] = NewTokenMode("default", + UseTokenType(a).WithSetMode(1), + UseTokenType(inDefault), + ) + modes[1] = NewTokenMode("second", + UseTokenType(inSecond), + UseTokenType(b).WithPopMode(), + ) + lexer := NewDefaultLexer(0, modes...) + + result := lexer.Exec("axbxy") + assert.Equal(t, []string{"A", "IN_SECOND", "B", "IN_SECOND"}, names(result.Tokens)) + require.Len(t, result.Errors, 1, "y belongs to the start mode, which is no longer active") + assert.Equal(t, int32(4), result.Errors[0].Range.Start) +} + +func TestExecSetModeKeepsOuterModeReachable(t *testing.T) { + // default -pushes-> middle -sets-> other -pops-> default + enter := literal(1, "ENTER", "(") + set := literal(2, "SET", "s") + leave := literal(3, "LEAVE", ")") + inMiddle := literal(4, "IN_MIDDLE", "m") + inOther := literal(5, "IN_OTHER", "o") + inDefault := literal(6, "IN_DEFAULT", "d") + + modes := make([]*TokenMode, 3) + modes[0] = NewTokenMode("default", + UseTokenType(enter).WithPushMode(1), + UseTokenType(inDefault), + ) + modes[1] = NewTokenMode("middle", + UseTokenType(inMiddle), + UseTokenType(set).WithSetMode(2), + ) + modes[2] = NewTokenMode("other", + UseTokenType(inOther), + UseTokenType(leave).WithPopMode(), + ) + lexer := NewDefaultLexer(0, modes...) + + result := lexer.Exec("d(mso)d") + assert.Equal(t, []string{ + "IN_DEFAULT", "ENTER", "IN_MIDDLE", "SET", "IN_OTHER", "LEAVE", "IN_DEFAULT", + }, names(result.Tokens)) + assert.Empty(t, result.Errors) +} + +func TestExecUnbalancedPopStaysInStartMode(t *testing.T) { + // A pop in the start mode has nothing to return to and must not break + // lexing of the remaining input. + pop := literal(1, "POP", ")") + word := matchRunes(2, "WORD", lowercase) + lexer := NewDefaultLexer(0, NewTokenMode("default", + UseTokenType(pop).WithPopMode(), + UseTokenType(word), + )) + + result := lexer.Exec(")ab)cd") + assert.Equal(t, []string{"POP", "WORD", "POP", "WORD"}, names(result.Tokens)) + assert.Empty(t, result.Errors) +} + +func TestExecPopAndGroupApplyTogether(t *testing.T) { + // The token that leaves the mode is itself routed to the comment group. + enter := literal(1, "ENTER", "(") + leave := literal(2, "LEAVE", ")") + inside := matchRunes(3, "INSIDE", lowercase) + outside := literal(4, "OUTSIDE", "!") + + modes := make([]*TokenMode, 2) + modes[0] = NewTokenMode("default", + UseTokenType(enter).WithPushMode(1), + UseTokenType(outside), + ) + modes[1] = NewTokenMode("inner", + UseTokenType(leave).WithPopMode().WithModifier(core.CommentModifier), + UseTokenType(inside), + ) + lexer := NewDefaultLexer(0, modes...) + + result := lexer.Exec("(ab)!") + assert.Equal(t, []string{"ENTER", "INSIDE", "OUTSIDE"}, names(result.Tokens)) + assert.Equal(t, []string{")"}, images(result.Comments)) +} + +// --- Error handling --- + +func TestExecReportsUnmatchedInput(t *testing.T) { + word := matchRunes(1, "WORD", lowercase) + lexer := NewDefaultLexer(0, NewTokenMode("default", UseTokenType(word))) + + result := lexer.Exec("ab!cd") + assert.Equal(t, []string{"ab", "cd"}, images(result.Tokens)) + require.Len(t, result.Errors, 1) + assert.Equal(t, int32(2), result.Errors[0].Range.Start) + assert.Equal(t, int32(3), result.Errors[0].Range.End) +} + +func TestExecUnmatchedInputConsumesWholeRune(t *testing.T) { + // A multi-byte rune that matches nothing must be skipped as one unit, so + // the next offset stays on a rune boundary. + word := matchRunes(1, "WORD", lowercase) + lexer := NewDefaultLexer(0, NewTokenMode("default", UseTokenType(word))) + + result := lexer.Exec("aä b") + require.Len(t, result.Errors, 2) + assert.Equal(t, int32(1), result.Errors[0].Range.Start) + assert.Equal(t, int32(3), result.Errors[0].Range.End, "the two-byte rune must be consumed as a whole") + assert.Equal(t, []string{"a", "b"}, images(result.Tokens)) +} + +func TestExecUnmatchedInputKeepsActiveMode(t *testing.T) { + // An unrecognized character inside a pushed mode must not change the mode: + // the closing quote still pops. + lexer := stringLexer(0) + + result := lexer.Exec(`"in#side" ab`) + assert.Equal(t, []string{"QUOTE", "TEXT", "TEXT", "QUOTE", "ID"}, names(result.Tokens)) + require.Len(t, result.Errors, 1) + assert.Equal(t, int32(3), result.Errors[0].Range.Start) +} + +func TestExecUnterminatedModeAtEndOfInput(t *testing.T) { + lexer := stringLexer(0) + + result := lexer.Exec(`ab "unterminated`) + assert.Equal(t, []string{"ID", "QUOTE", "TEXT"}, names(result.Tokens)) + assert.Equal(t, []string{"ab", `"`, "unterminated"}, images(result.Tokens)) + // Leaving a mode open is not a lexer error; the parser reports the + // unfinished construct. + assert.Empty(t, result.Errors) +} + +func TestExecEmptyInput(t *testing.T) { + lexer := stringLexer(0) + + result := lexer.Exec("") + assert.Empty(t, result.Tokens) + assert.Empty(t, result.Comments) + assert.Empty(t, result.Errors) + assert.Nil(t, result.Modifiers) +} + +// --- Reuse of a single lexer instance --- + +func TestExecResetsModeStackBetweenRuns(t *testing.T) { + // A lexer is registered once per language and reused for every document, so + // input that ends inside a pushed mode must not affect the next run. + lexer := stringLexer(0) + + first := lexer.Exec(`"unterminated`) + require.Equal(t, []string{"QUOTE", "TEXT"}, names(first.Tokens)) + + second := lexer.Exec(`ab "in string" cd`) + assert.Equal(t, []string{"ID", "QUOTE", "TEXT", "QUOTE", "ID"}, names(second.Tokens)) + assert.Empty(t, second.Errors) +} + +func TestExecRepeatedRunsAreIdentical(t *testing.T) { + lexer := stringLexer(0) + const input = `ab "in string" cd` + + first := names(lexer.Exec(input).Tokens) + for range 5 { + assert.Equal(t, first, names(lexer.Exec(input).Tokens)) + } +} + +// TestExecIsSafeForConcurrentUse guards the shared lexer instance that the +// workspace builder uses to parse documents in parallel. Run with -race. +func TestExecIsSafeForConcurrentUse(t *testing.T) { + lexer := stringLexer(0) + inputs := []string{ + `ab "in string" cd`, + `"unterminated`, + `ab cd`, + `"a" "b" "c"`, + } + expected := make([][]string, len(inputs)) + for i, input := range inputs { + expected[i] = names(lexer.Exec(input).Tokens) + } + + var wg sync.WaitGroup + for round := range 50 { + for i, input := range inputs { + wg.Go(func() { + assert.Equal(t, expected[i], names(lexer.Exec(input).Tokens), + "round %d, input %q", round, input) + }) + } + } + wg.Wait() +} diff --git a/lexer/modes.go b/lexer/modes.go new file mode 100644 index 00000000..7a7f72e7 --- /dev/null +++ b/lexer/modes.go @@ -0,0 +1,125 @@ +package lexer + +import core "typefox.dev/fastbelt" + +// TokenTypeUsage describes how a [core.TokenType] is used in a [TokenMode]. +// A token type can be used once per token mode, but can have different usages in different token modes. +// // Token mode `B` can be pushed on the mode stack while lexing a token of a certain type `T` in +// mode `A`, while mode `B` can be popped on some token of the same type `T` within mode `B`. +type TokenTypeUsage struct { + TokenType *core.TokenType + PushMode int + PopMode bool + Modifier int +} + +type TokenMode struct { + Name string + TokenTypes []*TokenTypeUsage + TokenMap [][]*TokenTypeUsage +} + +func UseTokenType(tokenType *core.TokenType) *TokenTypeUsage { + return &TokenTypeUsage{ + TokenType: tokenType, + PushMode: -1, + PopMode: false, + Modifier: 0, + } +} + +func (ttu *TokenTypeUsage) WithPushMode(mode int) *TokenTypeUsage { + ttu.PushMode = mode + return ttu +} + +func (ttu *TokenTypeUsage) WithPopMode() *TokenTypeUsage { + ttu.PopMode = true + return ttu +} + +func (ttu *TokenTypeUsage) WithSetMode(mode int) *TokenTypeUsage { + ttu.PopMode = true + ttu.PushMode = mode + return ttu +} + +func (ttu *TokenTypeUsage) WithModifier(modifier int) *TokenTypeUsage { + ttu.Modifier = modifier + return ttu +} + +// IsSkipped reports whether t is routed to the skipped-token group. +func (t *TokenTypeUsage) IsSkipped() bool { + return t.Modifier == core.SkippedModifier +} + +// IsComment reports whether t is routed to the comment-token group. +func (t *TokenTypeUsage) IsComment() bool { + return t.Modifier == core.CommentModifier +} + +// NewTokenMode returns a token mode that recognizes the given token types. +// At each position the longest match wins; among equal-length matches, the +// first argument wins. +func NewTokenMode(name string, tokenTypeUsages ...*TokenTypeUsage) *TokenMode { + tokenMap := make([][]*TokenTypeUsage, maxChar) + for i := range maxChar { + tokenMap[i] = make([]*TokenTypeUsage, 0) + } + for _, tokenTypeUsage := range tokenTypeUsages { + for _, r := range tokenTypeUsage.TokenType.StartChars { + index := int(r) % maxChar + tokenMap[index] = append(tokenMap[index], tokenTypeUsage) + } + } + return &TokenMode{ + Name: name, + TokenTypes: tokenTypeUsages, + TokenMap: tokenMap, + } +} + +// TokenModeStack tracks which [TokenMode] a lexer run is currently in. The +// bottom entry is the mode the run started in and is never removed, so the stack +// is never empty. TokenModeStack is not safe for concurrent use; each +// [DefaultLexer.Exec] uses its own. +type TokenModeStack struct { + modes []*TokenMode +} + +// NewTokenModeStack returns a stack holding defaultMode as its only entry. +func NewTokenModeStack(defaultMode *TokenMode) *TokenModeStack { + return &TokenModeStack{ + modes: []*TokenMode{defaultMode}, + } +} + +// Push makes mode the active mode, keeping the previous one for a later [Pop]. +func (s *TokenModeStack) Push(mode *TokenMode) { + s.modes = append(s.modes, mode) +} + +// Pop removes the active mode and returns it, making the mode below it active +// again. Popping the bottom entry is a no-op that returns the start mode: input +// with more pops than pushes stays in the start mode rather than failing. +func (s *TokenModeStack) Pop() *TokenMode { + if len(s.modes) <= 1 { + return s.modes[0] + } + mode := s.modes[len(s.modes)-1] + s.modes = s.modes[:len(s.modes)-1] + return mode +} + +// Peek returns the active mode. +func (s *TokenModeStack) Peek() *TokenMode { + return s.modes[len(s.modes)-1] +} + +// SetMode replaces the active mode with mode, leaving the rest of the stack +// untouched. This is what a `mode(X)` command does: unlike [Push] it does not +// deepen the stack, so it cannot be undone by a later [Pop]. +func (s *TokenModeStack) SetMode(mode *TokenMode) { + s.modes[len(s.modes)-1] = mode +} diff --git a/lexer/modes_test.go b/lexer/modes_test.go new file mode 100644 index 00000000..5ea02614 --- /dev/null +++ b/lexer/modes_test.go @@ -0,0 +1,203 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package lexer + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + core "typefox.dev/fastbelt" +) + +// literal returns a token type that matches the exact string value. +func literal(id int, name, value string) *core.TokenType { + return core.NewTokenType(id, name, name, core.TokenKindKeyword, + func(input string, offset int) int { + if strings.HasPrefix(input[offset:], value) { + return len(value) + } + return 0 + }, + []rune(value)[:1], + ) +} + +// matchRunes returns a token type that matches a run of any of the given runes. +func matchRunes(id int, name string, runes string) *core.TokenType { + return core.NewTokenType(id, name, name, core.TokenKindToken, + func(input string, offset int) int { + length := 0 + for _, r := range input[offset:] { + if !strings.ContainsRune(runes, r) { + break + } + length += len(string(r)) + } + return length + }, + []rune(runes), + ) +} + +// --- NewTokenMode: start character bucketing --- + +// Start characters are bucketed by rune value modulo maxChar (256), +// so that the lexer can quickly find candidate token types. +// Each mode has its own bucket map (depends on the used token types and +// their registration order). We test the candidate lookup here. + +func TestNewTokenModeBucketsByStartChar(t *testing.T) { + a := literal(1, "A", "a") + b := literal(2, "B", "b") + mode := NewTokenMode("test", UseTokenType(a), UseTokenType(b)) + + require.Len(t, mode.TokenMap, maxChar) + require.Len(t, mode.TokenMap['a'], 1) + assert.Same(t, a, mode.TokenMap['a'][0].TokenType) + require.Len(t, mode.TokenMap['b'], 1) + assert.Same(t, b, mode.TokenMap['b'][0].TokenType) + assert.Empty(t, mode.TokenMap['c']) +} + +func TestNewTokenModeSharedStartChar(t *testing.T) { + short := literal(1, "Short", "a") + long := literal(2, "Long", "ab") + mode := NewTokenMode("test", UseTokenType(short), UseTokenType(long)) + + // Both are candidates for 'a' and stay in registration order. + require.Len(t, mode.TokenMap['a'], 2) + assert.Same(t, short, mode.TokenMap['a'][0].TokenType) + assert.Same(t, long, mode.TokenMap['a'][1].TokenType) +} + +func TestNewTokenModeMultipleStartChars(t *testing.T) { + digits := matchRunes(1, "DIGITS", "0123456789") + mode := NewTokenMode("test", UseTokenType(digits)) + + for _, r := range "0123456789" { + assert.Len(t, mode.TokenMap[r], 1, "expected candidate for %q", r) + } + assert.Empty(t, mode.TokenMap['a']) +} + +func TestNewTokenModeWithoutStartCharsIsNeverACandidate(t *testing.T) { + // A token type without start characters cannot be preselected, so it is + // unreachable no matter what the input is. + unreachable := core.NewTokenType(1, "NONE", "NONE", core.TokenKindToken, + func(string, int) int { return 1 }, nil) + mode := NewTokenMode("test", UseTokenType(unreachable)) + + for index := range maxChar { + assert.Empty(t, mode.TokenMap[index]) + } + // It is still registered on the mode itself. + require.Len(t, mode.TokenTypes, 1) +} + +func TestNewTokenModeNonAsciiStartCharWrapsIntoBucket(t *testing.T) { + // Start characters are bucketed by rune value modulo maxChar, so a non-ASCII + // rune shares a bucket with the ASCII rune it collides with. Both must + // remain candidates for their own input. + umlaut := literal(1, "UMLAUT", "ä") // U+00E4, 0xE4 % 256 == 0xE4 + mode := NewTokenMode("test", UseTokenType(umlaut)) + + require.Len(t, mode.TokenMap[0xE4], 1) + assert.Same(t, umlaut, mode.TokenMap[0xE4][0].TokenType) +} + +func TestNewTokenModeCollidingStartCharsKeepBothCandidates(t *testing.T) { + // U+0161 % 256 == 0x61 == 'a', so these two share a bucket. + ascii := literal(1, "ASCII", "a") + wrapped := literal(2, "WRAPPED", "š") + mode := NewTokenMode("test", UseTokenType(ascii), UseTokenType(wrapped)) + + require.Len(t, mode.TokenMap['a'], 2) +} + +// --- TokenTypeUsage builders --- + +func TestUseTokenTypeDefaults(t *testing.T) { + usage := UseTokenType(literal(1, "A", "a")) + assert.Equal(t, -1, usage.PushMode) + assert.False(t, usage.PopMode) + assert.Equal(t, 0, usage.Modifier) + assert.False(t, usage.IsSkipped()) + assert.False(t, usage.IsComment()) +} + +func TestTokenTypeUsageBuilders(t *testing.T) { + tokenType := literal(1, "A", "a") + + push := UseTokenType(tokenType).WithPushMode(3) + assert.Equal(t, 3, push.PushMode) + assert.False(t, push.PopMode) + + pop := UseTokenType(tokenType).WithPopMode() + assert.True(t, pop.PopMode) + assert.Equal(t, -1, pop.PushMode) + + set := UseTokenType(tokenType).WithSetMode(2) + assert.True(t, set.PopMode) + assert.Equal(t, 2, set.PushMode) + + assert.True(t, UseTokenType(tokenType).WithModifier(core.SkippedModifier).IsSkipped()) + assert.True(t, UseTokenType(tokenType).WithModifier(core.CommentModifier).IsComment()) + assert.False(t, UseTokenType(tokenType).WithModifier(7).IsSkipped()) +} + +// --- TokenModeStack --- + +func TestTokenModeStackPushPopPeek(t *testing.T) { + first := NewTokenMode("first") + second := NewTokenMode("second") + stack := NewTokenModeStack(first) + + assert.Same(t, first, stack.Peek()) + stack.Push(second) + assert.Same(t, second, stack.Peek()) + assert.Same(t, second, stack.Pop()) + assert.Same(t, first, stack.Peek()) +} + +func TestTokenModeStackPopAtBottomIsNoOp(t *testing.T) { + first := NewTokenMode("first") + stack := NewTokenModeStack(first) + + // An unbalanced pop keeps the start mode active rather than emptying the + // stack, so lexing can continue. + assert.Same(t, first, stack.Pop()) + assert.Same(t, first, stack.Peek()) + assert.Same(t, first, stack.Pop()) + assert.Same(t, first, stack.Peek()) +} + +func TestTokenModeStackSetModeReplacesActiveMode(t *testing.T) { + first := NewTokenMode("first") + second := NewTokenMode("second") + third := NewTokenMode("third") + stack := NewTokenModeStack(first) + + stack.Push(second) + stack.SetMode(third) + assert.Same(t, third, stack.Peek()) + // SetMode replaced `second` instead of stacking on top of it, so popping + // returns to `first`. + assert.Same(t, third, stack.Pop()) + assert.Same(t, first, stack.Peek()) +} + +func TestTokenModeStackSetModeAtBottomKeepsDepth(t *testing.T) { + first := NewTokenMode("first") + second := NewTokenMode("second") + stack := NewTokenModeStack(first) + + stack.SetMode(second) + assert.Same(t, second, stack.Peek()) + // The stack is still one deep, so a pop cannot get back to `first`. + assert.Same(t, second, stack.Pop()) + assert.Same(t, second, stack.Peek()) +} diff --git a/parser/atn_simulator_test.go b/parser/atn_simulator_test.go index cc9c95dd..aa07d2bc 100644 --- a/parser/atn_simulator_test.go +++ b/parser/atn_simulator_test.go @@ -30,7 +30,7 @@ func linearATN(t *testing.T, tokenType *core.TokenType) *RuntimeATN { } func createTokenType(id int, name string) *core.TokenType { - return core.NewTokenType(id, name, name, 0, 0, 0, false, nil, nil) + return core.NewTokenType(id, name, name, 0, nil, nil) } // TestSimulator_AlternativeCoverage builds an ATN for `rule: 'a' 'x' | 'a' 'y'` diff --git a/parser/completion_parser_test.go b/parser/completion_parser_test.go index 46f10a4f..61edf59a 100644 --- a/parser/completion_parser_test.go +++ b/parser/completion_parser_test.go @@ -14,7 +14,7 @@ import ( // methods: EnterRule/ExitRule push and pop, RecordSnapshot/EnterRule both // append to Snapshots, and MarkAssignment updates the top frame. func TestCompletionParserState_SnapshotsAndStack(t *testing.T) { - tA := core.NewTokenType(1, "a", "a", 0, 0, 0, false, nil, nil) + tA := core.NewTokenType(1, "a", "a", 0, nil, nil) // Build a small ATN so NewParserState has something to hold. s0 := &RuntimeATNState{StateNumber: 0, Type: ATNBasic, Decision: -1} diff --git a/parser/error_recovery.go b/parser/error_recovery.go index 0e5ba3de..b39f05f5 100644 --- a/parser/error_recovery.go +++ b/parser/error_recovery.go @@ -67,6 +67,13 @@ func (DefaultErrorRecovery) Recover(parserState *ParserState) { if parserState.ErrorMode != ErrorModeRecover { return } + + //if we are stuck on the same token, consume it and try to move on + if parserState.lastErrorIndex == parserState.Index && parserState.Index < parserState.Length { + parserState.Index++ + } + parserState.lastErrorIndex = parserState.Index + // Discards tokens until we find one that the current follow set, or until we hit EOF. if !parserState.FollowSetEmpty() { for { diff --git a/parser/parser.go b/parser/parser.go index cf7c7310..5814c1e3 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -45,6 +45,8 @@ type ParserState struct { recovery ErrorRecoveryStrategy messages ErrorMessageProvider sim *parserATNSimulator // lazily created adaptive (ALL(*)) predictor + + lastErrorIndex int // index of the last token that caused an error, used to prevent infinite loops in recovery } func (p *ParserState) ATN() *RuntimeATN { @@ -124,15 +126,16 @@ func NewParserState(tokens []core.Token, atn *RuntimeATN, recovery ErrorRecovery panic("atn must be provided") } return &ParserState{ - Tokens: tokens, - Length: len(tokens), - Index: 0, - ErrorMode: ErrorModeNone, - errors: []*core.ParserError{}, - atn: atn, - followStates: nil, - recovery: recovery, - messages: messages, + Tokens: tokens, + Length: len(tokens), + Index: 0, + ErrorMode: ErrorModeNone, + errors: []*core.ParserError{}, + atn: atn, + followStates: nil, + recovery: recovery, + messages: messages, + lastErrorIndex: -1, } } diff --git a/server/document_sync_test.go b/server/document_sync_test.go index a52ea059..c70bee91 100644 --- a/server/document_sync_test.go +++ b/server/document_sync_test.go @@ -309,6 +309,7 @@ func TestTextDocuments_DidSave(t *testing.T) { doc := textdocStore.GetOverlay(uri) if doc == nil { t.Fatal("document should still exist after DidSave") + return } if doc.URI() != uri { t.Errorf("expected URI %s, got %s", uri, doc.URI()) diff --git a/token.go b/token.go index 085a301e..d6059417 100644 --- a/token.go +++ b/token.go @@ -6,15 +6,16 @@ package fastbelt import "typefox.dev/fastbelt/util/collections" -// SkippedGroup marks token types that the lexer should drop from all output streams. -const SkippedGroup = -1 +// SkippedModifier marks token types that the lexer should drop from all output streams. +const SkippedModifier = -1 -// CommentGroup marks token types that the lexer should collect in Document.Comments. -const CommentGroup = -2 +// CommentModifier marks token types that the lexer should collect in Document.Comments. +const CommentModifier = -2 -// TokenKind names the grammar construct that produced a TokenType. It is -// distinct from Group: Group controls lexer-stream behaviour (skipped / -// comment), while Kind describes the grammar origin and is consumed by +// TokenKind denotes the grammar construct that produced a TokenType: +// 'Keyword', 'Terminal', or 'Group'. It is distinct from Modifier: +// Modifier controls lexer-stream behaviour (skipped / comment), while +// Kind describes the grammar origin and is consumed by // downstream features such as the completion engine, which by default // only surfaces keyword-kind tokens as completion candidates. type TokenKind int @@ -23,7 +24,7 @@ const ( // TokenKindToken is the default - a TokenType produced by a named // `token` rule in the .fb grammar (regex-matched). Hidden and comment // tokens are also TokenKindToken; their stream behaviour is encoded - // separately in Group. + // separately in Modifier. TokenKindToken TokenKind = 0 // TokenKindKeyword is a TokenType produced by a literal string in a // parser rule (e.g. `"statemachine"`). Matched by a string prefix. @@ -53,14 +54,8 @@ type TokenType struct { Label string // StartChars contains candidate start runes used for lexer preselection. StartChars []rune - // Group controls lexer output routing (default stream, skipped, comments, or custom groups). - Group int // Kind records whether the token comes from a keyword literal or a token rule. Kind TokenKind - // PushMode selects the next lexer mode after this token is matched. - PushMode int - // PopMode reports whether matching this token pops one lexer mode. - PopMode bool // Match performs the actual token match at a given input offset. Match TokenMatcher // Matches returns whether the token type matches another, given type @@ -71,22 +66,19 @@ type TokenType struct { } // NewTokenType creates a token type descriptor used by generated lexers and parsers. -func NewTokenType(id int, name, label string, group int, kind TokenKind, pushMode int, popMode bool, match TokenMatcher, startChars []rune) *TokenType { +func NewTokenType(id int, name, label string, kind TokenKind, match TokenMatcher, startChars []rune) *TokenType { matching := collections.NewBitset() matching.Insert(id) tt := &TokenType{ Id: id, Name: name, Label: label, - Group: group, Kind: kind, Match: match, Matches: func(other *TokenType) bool { return other.Id == id }, bitset: matching, - PushMode: pushMode, - PopMode: popMode, StartChars: startChars, } tt.MatchingTokens = []*TokenType{tt} @@ -133,16 +125,6 @@ func unrollMatchingTokens(matchingTypes []*TokenType) []*TokenType { return unrolled } -// IsSkipped reports whether t is routed to the skipped-token group. -func (t *TokenType) IsSkipped() bool { - return t.Group == SkippedGroup -} - -// IsComment reports whether t is routed to the comment-token group. -func (t *TokenType) IsComment() bool { - return t.Group == CommentGroup -} - // IsKeyword reports whether t originates from a grammar keyword literal. func (t *TokenType) IsKeyword() bool { return t.Kind == TokenKindKeyword @@ -158,10 +140,7 @@ var EOF = NewTokenType( 0, "EOF", "EOF", - 0, TokenKindToken, - 0, - false, nil, nil, ) @@ -193,11 +172,6 @@ func NewToken(tokenType *TokenType, image string, startOffset, endOffset int) To } } -// IsSkipped reports whether t belongs to the skipped lexer group. -func (t *Token) IsSkipped() bool { - return t.Type != nil && t.Type.Group == SkippedGroup -} - // IsEOF reports whether t uses the end-of-input token type. func (t *Token) IsEOF() bool { return t.Type == EOF diff --git a/workspace/document_parser.go b/workspace/document_parser.go index f62f5144..c935c417 100644 --- a/workspace/document_parser.go +++ b/workspace/document_parser.go @@ -33,7 +33,7 @@ func (s *DefaultDocumentParser) Parse(doc *core.Document) { text := doc.TextDoc.Text(nil) // Run the lexer lexer := service.MustGet[lexer.Lexer](s.sc) - lexerRes := lexer.Lex(text) + lexerRes := lexer.Exec(text) doc.LexerErrors = lexerRes.Errors doc.Tokens = lexerRes.Tokens doc.Comments = lexerRes.Comments