Introduce token modes

.vscode/launch.json

@@ -92,6 +92,34 @@
 			"outFiles": [
 				"${workspaceFolder}/internal/vscode-extensions/statemachine/dist/**/*.js"
 			]
-		}
+		},
+		{
+			"name": "TokenModesWithGroups Generator",
+			"type": "go",
+			"request": "launch",
+			"mode": "debug",
+			"program": "${workspaceFolder}/cmd/fastbelt",
+			"cwd": "${workspaceFolder}/internal/languages/token_modes_with_groups",
+			"args": [
+				"generate",
+				"token_modes_with_groups.fb",
+				"-v",
+				"--atn"
+			]
+		},
+		{
+			"name": "Completion Generator",
+			"type": "go",
+			"request": "launch",
+			"mode": "debug",
+			"program": "${workspaceFolder}/cmd/fastbelt",
+			"cwd": "${workspaceFolder}/internal/languages/completion",
+			"args": [
+				"generate",
+				"completion.fb",
+				"-v",
+				"--atn"
+			]
+		},
 	]
 }

examples/arithmetics/benchmark_test.go

@@ -35,7 +35,7 @@ func BenchmarkParser(b *testing.B) {
 	srv := CreateServices()
 	lexerService := service.MustGet[lexer.Lexer](srv)
 	parserService := service.MustGet[parser.Parser](srv)
-	lexed := lexerService.Lex(content)
+	lexed := lexerService.Exec(content)
 	if len(lexed.Errors) > 0 {
 		b.Fatalf("lexer errors: %v", lexed.Errors)
 	}

examples/statemachine/benchmark_test.go

@@ -127,7 +127,7 @@ func BenchmarkParser(b *testing.B) {
 	srv := CreateServices()
 	lexerService := service.MustGet[lexer.Lexer](srv)
 	parserService := service.MustGet[parser.Parser](srv)
-	tokens := lexerService.Lex(content).Tokens
+	tokens := lexerService.Exec(content).Tokens
 	doc, err := fastbelt.NewDocumentFromString("file:///workspace/statemachine_0.statemachine", "statemachine", content)
 	if err != nil {
 		b.Fatal(err)
@@ -148,7 +148,7 @@ func BenchmarkLexer(b *testing.B) {
 	b.SetBytes(int64(len(content)))
 	b.ResetTimer()
 	for b.Loop() {
-		_ = l.Lex(content)
+		_ = l.Exec(content)
 	}
 }
 
@@ -165,7 +165,7 @@ func BenchmarkLexerAndParser(b *testing.B) {
 	b.SetBytes(int64(len(content)))
 	b.ResetTimer()
 	for b.Loop() {
-		doc.Tokens = lexerService.Lex(content).Tokens
+		doc.Tokens = lexerService.Exec(content).Tokens
 		result := parserService.Parse(doc)
 		doc.Root = result.Node
 	}