From 25ea397c9b9d7dd689dbb7747531a36cbc236856 Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Fri, 20 Mar 2026 16:42:52 +0100 Subject: [PATCH 01/12] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20LSP=20server=20?= =?UTF-8?q?for=20integration=20package=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrap existing package-spec validation in an LSP server (stdio) so editors can surface lint errors inline on open/save. Uses tliron/glsp for the JSON-RPC transport. Includes debounced per-package validation, error-to-file routing, and diagnostic clearing. Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/lsp.go | 48 +++++++++ cmd/root.go | 1 + go.mod | 13 ++- go.sum | 27 ++++- internal/lsp/debounce.go | 50 ++++++++++ internal/lsp/diagnostics.go | 143 +++++++++++++++++++++++++++ internal/lsp/diagnostics_test.go | 80 +++++++++++++++ internal/lsp/server.go | 164 +++++++++++++++++++++++++++++++ internal/lsp/workspace.go | 51 ++++++++++ 9 files changed, 571 insertions(+), 6 deletions(-) create mode 100644 cmd/lsp.go create mode 100644 internal/lsp/debounce.go create mode 100644 internal/lsp/diagnostics.go create mode 100644 internal/lsp/diagnostics_test.go create mode 100644 internal/lsp/server.go create mode 100644 internal/lsp/workspace.go diff --git a/cmd/lsp.go b/cmd/lsp.go new file mode 100644 index 0000000000..4bca9bf703 --- /dev/null +++ b/cmd/lsp.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/tliron/commonlog" + _ "github.com/tliron/commonlog/simple" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/lsp" +) + +const lspLongDescription = `Start a Language Server Protocol (LSP) server for Elastic integration packages. + +The LSP server communicates over stdin/stdout and provides real-time validation +diagnostics for integration packages opened in supported editors.` + +func setupLSPCommand() *cobraext.Command { + cmd := &cobra.Command{ + Use: "lsp", + Short: "Start the LSP server", + Long: lspLongDescription, + Args: cobra.NoArgs, + RunE: lspCommandAction, + // Override the parent's PersistentPreRunE to prevent version check + // messages and install output from corrupting the JSON-RPC stdio stream. + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + cmd.Flags().String("log-file", "", "Path to log file for debug output") + + return cobraext.NewCommand(cmd, cobraext.ContextGlobal) +} + +func lspCommandAction(cmd *cobra.Command, args []string) error { + logFile, _ := cmd.Flags().GetString("log-file") + if logFile != "" { + commonlog.Configure(2, &logFile) + } + + s := lsp.NewServer() + return s.RunStdio() +} diff --git a/cmd/root.go b/cmd/root.go index c5f29569bc..b20673d8cb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -32,6 +32,7 @@ var commands = []*cobraext.Command{ setupInstallCommand(), setupLinksCommand(), setupLintCommand(), + setupLSPCommand(), setupModifyCommand(), setupProfilesCommand(), setupReportsCommand(), diff --git a/go.mod b/go.mod index 9012502d22..dff700e875 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/charmbracelet/bubbles v1.0.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 - github.com/creack/pty v1.1.19 + github.com/creack/pty v1.1.24 github.com/dustin/go-humanize v1.0.1 github.com/elastic/elastic-integration-corpus-generator-tool v0.12.0 github.com/elastic/go-elasticsearch/v7 v7.17.10 @@ -38,6 +38,8 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 + github.com/tliron/commonlog v0.2.21 + github.com/tliron/glsp v0.2.2 go.yaml.in/yaml/v2 v2.4.4 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/tools v0.43.0 @@ -100,11 +102,13 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.5.0 // indirect + github.com/iancoleman/strcase v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -136,19 +140,24 @@ require ( github.com/olekukonko/ll v0.1.6 // indirect github.com/oschwald/maxminddb-golang/v2 v2.1.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe // indirect github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sahilm/fuzzy v0.1.1 // indirect + github.com/sasha-s/go-deadlock v0.3.6 // indirect + github.com/segmentio/ksuid v1.0.4 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.4.0 // indirect + github.com/sourcegraph/jsonrpc2 v0.2.0 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tliron/go-kutil v0.4.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect @@ -169,7 +178,7 @@ require ( golang.org/x/term v0.41.0 // indirect golang.org/x/text v0.35.0 // indirect golang.org/x/time v0.12.0 // indirect - google.golang.org/protobuf v1.36.8 // indirect + google.golang.org/protobuf v1.36.9 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 8c557195c0..a3befb2e02 100644 --- a/go.sum +++ b/go.sum @@ -73,8 +73,8 @@ github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.19 h1:tUN6H7LWqNx4hQVxomd0CVsDwaDr9gaRQaI4GpSmrsA= -github.com/creack/pty v1.1.19/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -171,6 +171,9 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -186,6 +189,8 @@ github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVU github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= @@ -280,6 +285,8 @@ github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe h1:vHpqOnPlnkba8iSxU4j/CvDSS9J4+F4473esQsYLGoE= +github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -295,6 +302,10 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/sasha-s/go-deadlock v0.3.6 h1:TR7sfOnZ7x00tWPfD397Peodt57KzMDo+9Ae9rMiUmw= +github.com/sasha-s/go-deadlock v0.3.6/go.mod h1:CUqNyyvMxTyjFqDT7MRg9mb4Dv/btmGTqSR+rky/UXo= +github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= +github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= @@ -306,6 +317,8 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sourcegraph/jsonrpc2 v0.2.0 h1:KjN/dC4fP6aN9030MZCJs9WQbTOjWHhrtKVpzzSrr/U= +github.com/sourcegraph/jsonrpc2 v0.2.0/go.mod h1:ZafdZgk/axhT1cvZAPOhw+95nz2I/Ra5qMlU4gTRwIo= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= @@ -335,6 +348,12 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tliron/commonlog v0.2.21 h1:V1v+6opmzuOqDxxnxxM5RWtlHZmqZlDxkKeZGs6DpPg= +github.com/tliron/commonlog v0.2.21/go.mod h1:W6XVoS/zo7mHXv2Kz8HKnBq+U34dFysJ2KUh2Aboibw= +github.com/tliron/glsp v0.2.2 h1:IKPfwpE8Lu8yB6Dayta+IyRMAbTVunudeauEgjXBt+c= +github.com/tliron/glsp v0.2.2/go.mod h1:GMVWDNeODxHzmDPvYbYTCs7yHVaEATfYtXiYJ9w1nBg= +github.com/tliron/go-kutil v0.4.0 h1:5JwcBacgnqS3XyhwCWZKvq8ftlbVttNXnt+kfCH+Y2E= +github.com/tliron/go-kutil v0.4.0/go.mod h1:hpHVq+CP1uci2M208UEjPiPwsRsz/QweGBnLB3CaQ24= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -440,8 +459,8 @@ golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= -google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= +google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/lsp/debounce.go b/internal/lsp/debounce.go new file mode 100644 index 0000000000..88b5214cbb --- /dev/null +++ b/internal/lsp/debounce.go @@ -0,0 +1,50 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "sync" + "time" +) + +const debounceDelay = 500 * time.Millisecond + +// Debouncer coalesces rapid validation triggers per package root. +type Debouncer struct { + mu sync.Mutex + timers map[string]*time.Timer +} + +// NewDebouncer creates a new Debouncer. +func NewDebouncer() *Debouncer { + return &Debouncer{ + timers: make(map[string]*time.Timer), + } +} + +// Trigger schedules fn to run after the debounce delay. If Trigger is called +// again for the same key before the delay elapses, the previous call is +// cancelled and the timer resets. +func (d *Debouncer) Trigger(key string, fn func()) { + d.mu.Lock() + defer d.mu.Unlock() + + if t, ok := d.timers[key]; ok { + t.Stop() + } + + d.timers[key] = time.AfterFunc(debounceDelay, fn) +} + +// Shutdown stops all pending timers. +func (d *Debouncer) Shutdown() { + d.mu.Lock() + defer d.mu.Unlock() + + for key, t := range d.timers { + t.Stop() + delete(d.timers, key) + } +} diff --git a/internal/lsp/diagnostics.go b/internal/lsp/diagnostics.go new file mode 100644 index 0000000000..4ef66ee1a4 --- /dev/null +++ b/internal/lsp/diagnostics.go @@ -0,0 +1,143 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" + protocol "github.com/tliron/glsp/protocol_3_16" + + "github.com/elastic/elastic-package/internal/validation" +) + +var ( + // Matches: file "path/to/file" is invalid: message + fileErrorRe = regexp.MustCompile(`^file "([^"]+)" is invalid: (.+)$`) + + // Matches folder-based errors like: + // "expecting to find [X] folder in folder [/path/to/dir]" + // "expecting to find [X] file in folder [/path/to/dir]" + folderErrorRe = regexp.MustCompile(`in folder \[([^\]]+)\]`) + + // Matches error code suffix like (SVR00001) + errorCodeRe = regexp.MustCompile(`\(([A-Z]{2,3}\d{5})\)$`) +) + +// validatePackage runs validation on the package at the given root path and +// returns diagnostics grouped by absolute file path. +func validatePackage(packageRoot string) map[string][]protocol.Diagnostic { + diagsByFile := make(map[string][]protocol.Diagnostic) + + errs, _ := validation.ValidateAndFilterFromPath(packageRoot) + if errs == nil { + // No errors — return empty map (will clear diagnostics). + // Include the manifest so we clear any previous diagnostics for it. + manifestPath := filepath.Join(packageRoot, "manifest.yml") + diagsByFile[manifestPath] = []protocol.Diagnostic{} + return diagsByFile + } + + // Split into individual errors. + var individualErrors []string + if ve, ok := errs.(specerrors.ValidationErrors); ok { + for _, e := range ve { + individualErrors = append(individualErrors, e.Error()) + } + } else { + // Single error or unknown type — split on newlines as fallback. + for _, line := range strings.Split(errs.Error(), "\n") { + line = strings.TrimSpace(line) + if line != "" && !strings.HasPrefix(line, "found ") { + // Strip leading numbering like " 1. " + line = stripNumbering(line) + if line != "" { + individualErrors = append(individualErrors, line) + } + } + } + } + + for _, errMsg := range individualErrors { + filePath, message, code := parseError(errMsg, packageRoot) + + diag := protocol.Diagnostic{ + Range: protocol.Range{ + Start: protocol.Position{Line: 0, Character: 0}, + End: protocol.Position{Line: 0, Character: 0}, + }, + Severity: diagnosticSeverityPtr(protocol.DiagnosticSeverityError), + Source: strPtr(serverName), + Message: message, + } + + if code != "" { + diag.Code = &protocol.IntegerOrString{Value: code} + } + + diagsByFile[filePath] = append(diagsByFile[filePath], diag) + } + + // Ensure we publish empty diagnostics for files that previously had errors + // but are now clean. The caller handles this by tracking state. + // For now, we only publish files that have errors. + + return diagsByFile +} + +// parseError extracts the file path, message, and error code from an error string. +func parseError(errMsg string, packageRoot string) (filePath, message, code string) { + // Try to extract error code. + if m := errorCodeRe.FindStringSubmatch(errMsg); m != nil { + code = m[1] + } + + // Try the "file X is invalid: Y" pattern. + if m := fileErrorRe.FindStringSubmatch(errMsg); m != nil { + rawPath := m[1] + message = m[2] + + // The path may be absolute or relative to the package root. + if filepath.IsAbs(rawPath) { + filePath = rawPath + } else { + filePath = filepath.Join(packageRoot, rawPath) + } + return filePath, message, code + } + + // Try folder-based pattern: attribute to manifest.yml in that folder. + if m := folderErrorRe.FindStringSubmatch(errMsg); m != nil { + dir := m[1] + candidate := filepath.Join(dir, "manifest.yml") + if _, statErr := os.Stat(candidate); statErr == nil { + filePath = candidate + message = errMsg + return filePath, message, code + } + } + + // Fallback: attribute to manifest.yml. + filePath = filepath.Join(packageRoot, "manifest.yml") + message = errMsg + return filePath, message, code +} + +// stripNumbering removes leading numbering like " 1. " from a line. +func stripNumbering(s string) string { + re := regexp.MustCompile(`^\s*\d+\.\s+`) + return re.ReplaceAllString(s, "") +} + +func diagnosticSeverityPtr(s protocol.DiagnosticSeverity) *protocol.DiagnosticSeverity { + return &s +} + +func strPtr(s string) *string { + return &s +} diff --git a/internal/lsp/diagnostics_test.go b/internal/lsp/diagnostics_test.go new file mode 100644 index 0000000000..c2fab2cef4 --- /dev/null +++ b/internal/lsp/diagnostics_test.go @@ -0,0 +1,80 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseError_FilePattern(t *testing.T) { + packageRoot := "/home/user/packages/apache" + + filePath, message, code := parseError( + `file "/home/user/packages/apache/data_stream/access/fields/fields.yml" is invalid: field vars.1: Must not be present`, + packageRoot, + ) + + assert.Equal(t, "/home/user/packages/apache/data_stream/access/fields/fields.yml", filePath) + assert.Equal(t, "field vars.1: Must not be present", message) + assert.Equal(t, "", code) +} + +func TestParseError_RelativePath(t *testing.T) { + packageRoot := "/home/user/packages/apache" + + filePath, message, code := parseError( + `file "data_stream/access/manifest.yml" is invalid: field title: String length must be greater than or equal to 1`, + packageRoot, + ) + + assert.Equal(t, "/home/user/packages/apache/data_stream/access/manifest.yml", filePath) + assert.Equal(t, "field title: String length must be greater than or equal to 1", message) + assert.Equal(t, "", code) +} + +func TestParseError_WithCode(t *testing.T) { + packageRoot := "/home/user/packages/apache" + + filePath, message, code := parseError( + `changelog entry found for version 1.0.0 but package version is 2.0.0 (SVR00003)`, + packageRoot, + ) + + // No file pattern match → falls back to manifest.yml + assert.Equal(t, "/home/user/packages/apache/manifest.yml", filePath) + assert.Equal(t, "changelog entry found for version 1.0.0 but package version is 2.0.0 (SVR00003)", message) + assert.Equal(t, "SVR00003", code) +} + +func TestParseError_NoFilePattern(t *testing.T) { + packageRoot := "/home/user/packages/apache" + + filePath, message, code := parseError( + `some generic validation error`, + packageRoot, + ) + + assert.Equal(t, "/home/user/packages/apache/manifest.yml", filePath) + assert.Equal(t, "some generic validation error", message) + assert.Equal(t, "", code) +} + +func TestStripNumbering(t *testing.T) { + assert.Equal(t, "hello world", stripNumbering(" 1. hello world")) + assert.Equal(t, "error msg", stripNumbering(" 12. error msg")) + assert.Equal(t, "no number", stripNumbering("no number")) +} + +func TestURIConversion(t *testing.T) { + path := "/home/user/packages/apache/manifest.yml" + uri := pathToURI(path) + assert.Equal(t, "file:///home/user/packages/apache/manifest.yml", uri) + + roundTripped, err := uriToPath(uri) + assert.NoError(t, err) + assert.Equal(t, path, roundTripped) +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go new file mode 100644 index 0000000000..f9f4cb969f --- /dev/null +++ b/internal/lsp/server.go @@ -0,0 +1,164 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "sync" + + "github.com/tliron/commonlog" + "github.com/tliron/glsp" + protocol "github.com/tliron/glsp/protocol_3_16" + "github.com/tliron/glsp/server" +) + +const serverName = "elastic-package-lsp" + +var version = "0.1.0" + +// Server is the elastic-package LSP server. +type Server struct { + handler protocol.Handler + server *server.Server + debouncer *Debouncer + logger commonlog.Logger + + // prevDiagFiles tracks which files had diagnostics published in the last + // validation run per package root, so we can clear them when errors are fixed. + prevDiagFilesMu sync.Mutex + prevDiagFiles map[string]map[string]struct{} // packageRoot -> set of filePaths +} + +// NewServer creates a new LSP server. +func NewServer() *Server { + s := &Server{ + debouncer: NewDebouncer(), + logger: commonlog.GetLogger(serverName), + prevDiagFiles: make(map[string]map[string]struct{}), + } + + s.handler = protocol.Handler{ + Initialize: s.initialize, + Initialized: s.initialized, + Shutdown: s.shutdown, + SetTrace: s.setTrace, + TextDocumentDidOpen: s.textDocumentDidOpen, + TextDocumentDidSave: s.textDocumentDidSave, + TextDocumentDidClose: s.textDocumentDidClose, + } + + s.server = server.NewServer(&s.handler, serverName, false) + + return s +} + +// RunStdio starts the server on stdin/stdout. +func (s *Server) RunStdio() error { + return s.server.RunStdio() +} + +func (s *Server) initialize(ctx *glsp.Context, params *protocol.InitializeParams) (any, error) { + capabilities := s.handler.CreateServerCapabilities() + + sync := protocol.TextDocumentSyncKindFull + capabilities.TextDocumentSync = &protocol.TextDocumentSyncOptions{ + OpenClose: boolPtr(true), + Change: &sync, + Save: &protocol.SaveOptions{ + IncludeText: boolPtr(false), + }, + } + + return protocol.InitializeResult{ + Capabilities: capabilities, + ServerInfo: &protocol.InitializeResultServerInfo{ + Name: serverName, + Version: &version, + }, + }, nil +} + +func (s *Server) initialized(ctx *glsp.Context, params *protocol.InitializedParams) error { + return nil +} + +func (s *Server) shutdown(ctx *glsp.Context) error { + protocol.SetTraceValue(protocol.TraceValueOff) + s.debouncer.Shutdown() + return nil +} + +func (s *Server) setTrace(ctx *glsp.Context, params *protocol.SetTraceParams) error { + protocol.SetTraceValue(params.Value) + return nil +} + +func (s *Server) textDocumentDidOpen(ctx *glsp.Context, params *protocol.DidOpenTextDocumentParams) error { + s.triggerValidation(ctx, params.TextDocument.URI) + return nil +} + +func (s *Server) textDocumentDidSave(ctx *glsp.Context, params *protocol.DidSaveTextDocumentParams) error { + s.triggerValidation(ctx, params.TextDocument.URI) + return nil +} + +func (s *Server) textDocumentDidClose(ctx *glsp.Context, params *protocol.DidCloseTextDocumentParams) error { + return nil +} + +func (s *Server) triggerValidation(ctx *glsp.Context, uri protocol.DocumentUri) { + filePath, err := uriToPath(uri) + if err != nil { + s.logger.Errorf("failed to parse URI %s: %v", uri, err) + return + } + + packageRoot, err := findPackageRoot(filePath) + if err != nil { + // File is not inside a package — nothing to validate. + return + } + + s.debouncer.Trigger(packageRoot, func() { + diags := validatePackage(packageRoot) + s.publishAllDiagnostics(ctx, packageRoot, diags) + }) +} + +func (s *Server) publishAllDiagnostics(ctx *glsp.Context, packageRoot string, diagsByFile map[string][]protocol.Diagnostic) { + s.prevDiagFilesMu.Lock() + defer s.prevDiagFilesMu.Unlock() + + // Clear diagnostics for files that had errors before but don't anymore. + if prev, ok := s.prevDiagFiles[packageRoot]; ok { + for filePath := range prev { + if _, stillHasErrors := diagsByFile[filePath]; !stillHasErrors { + uri := pathToURI(filePath) + ctx.Notify(protocol.ServerTextDocumentPublishDiagnostics, protocol.PublishDiagnosticsParams{ + URI: uri, + Diagnostics: []protocol.Diagnostic{}, + }) + } + } + } + + // Publish current diagnostics and track the files. + currentFiles := make(map[string]struct{}, len(diagsByFile)) + for filePath, diags := range diagsByFile { + uri := pathToURI(filePath) + ctx.Notify(protocol.ServerTextDocumentPublishDiagnostics, protocol.PublishDiagnosticsParams{ + URI: uri, + Diagnostics: diags, + }) + if len(diags) > 0 { + currentFiles[filePath] = struct{}{} + } + } + s.prevDiagFiles[packageRoot] = currentFiles +} + +func boolPtr(b bool) *bool { + return &b +} diff --git a/internal/lsp/workspace.go b/internal/lsp/workspace.go new file mode 100644 index 0000000000..c79f5a0014 --- /dev/null +++ b/internal/lsp/workspace.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "fmt" + "net/url" + "path/filepath" + "runtime" + "strings" + + "github.com/elastic/elastic-package/internal/packages" +) + +// findPackageRoot finds the integration package root for a given file path. +// Returns ErrPackageRootNotFound if the file is not inside a package. +func findPackageRoot(filePath string) (string, error) { + dir := filepath.Dir(filePath) + return packages.FindPackageRootFrom(dir) +} + +// uriToPath converts a file:// URI to a filesystem path. +func uriToPath(uri string) (string, error) { + u, err := url.Parse(string(uri)) + if err != nil { + return "", fmt.Errorf("failed to parse URI: %w", err) + } + if u.Scheme != "file" { + return "", fmt.Errorf("unsupported URI scheme: %s", u.Scheme) + } + + path := u.Path + // On Windows, file URIs look like file:///C:/path, so we need to strip + // the leading slash from the path. + if runtime.GOOS == "windows" && strings.HasPrefix(path, "/") && len(path) > 2 && path[2] == ':' { + path = path[1:] + } + + return filepath.FromSlash(path), nil +} + +// pathToURI converts a filesystem path to a file:// URI. +func pathToURI(path string) string { + path = filepath.ToSlash(path) + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return "file://" + path +} From 2beb3d7d9ea92e8ea0b9347f5d2feeff64227e97 Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Fri, 20 Mar 2026 16:42:59 +0100 Subject: [PATCH 02/12] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20LSP=20comman?= =?UTF-8?q?d=20docs,=20demo=20script,=20and=20Zed=20extension=20install=20?= =?UTF-8?q?guide?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 87 +++++++++++++++++++++++++++++ scripts/lsp-demo.sh | 131 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100755 scripts/lsp-demo.sh diff --git a/README.md b/README.md index fb7e44fd17..e5bb4e2aab 100644 --- a/README.md +++ b/README.md @@ -439,6 +439,93 @@ Use this command to validate the contents of a package using the package specifi The command ensures that the package is aligned with the package spec and the README file is up-to-date with its template (if present). +### `elastic-package lsp` + +_Context: global_ + +Start a Language Server Protocol (LSP) server for Elastic integration packages. + +The LSP server communicates over stdin/stdout and provides real-time validation +diagnostics for integration packages opened in supported editors. When you open +or save any file inside a package, the server runs the same validation as +`elastic-package lint` and pushes diagnostics (errors) back to your editor. + +Flags: +- `--log-file ` — write debug logs to a file (useful for troubleshooting). + +#### Demo + +Run the included demo script to see the LSP in action without an editor: + +``` +$ ./scripts/lsp-demo.sh ./elastic-package + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Scenario 1: Valid package (apache) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + File: manifest.yml + (no errors) + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Scenario 2: Broken package (invalid name, missing files) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + File: data_stream/mystream/manifest.yml + [ERROR] expecting to find [fields] folder in ... + + File: manifest.yml + [ERROR] field owner: type is required + [ERROR] field name: Does not match pattern '^[a-z0-9_]+$' + [ERROR] expecting to find [changelog.yml] file in ... + [ERROR] expecting to find [docs] folder in ... + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Scenario 3: Open data_stream file — LSP discovers package root +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + (same errors as Scenario 2 — the LSP walks up to find the package root) +``` + +The script creates a temporary broken package, sends JSON-RPC messages to the +LSP, and pretty-prints the diagnostics it receives back. + +#### Zed editor extension + +A companion Zed extension is available at +[`zed-elastic-package`](https://github.com/elastic/zed-elastic-package). To +install it for development: + +1. Build elastic-package with the `lsp` command and ensure it is on your `PATH`: + ``` + go build -o elastic-package . + ``` + +2. Clone the extension repo: + ``` + git clone https://github.com/elastic/zed-elastic-package.git + ``` + +3. In Zed, open the command palette and run **zed: Install Dev Extension**, + then select the `zed-elastic-package` directory. + +4. Open any integration package. YAML and JSON files will automatically get + validation diagnostics from the LSP. + +To point the extension at a custom binary path, add this to your Zed settings: + +```json +{ + "lsp": { + "elastic-package-lsp": { + "binary": { + "path": "/path/to/elastic-package" + } + } + } +} +``` + ### `elastic-package modify` _Context: package_ diff --git a/scripts/lsp-demo.sh b/scripts/lsp-demo.sh new file mode 100755 index 0000000000..b4a77f3ef9 --- /dev/null +++ b/scripts/lsp-demo.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# Demo script for elastic-package LSP +# Usage: ./scripts/lsp-demo.sh [path-to-binary] +# +# Runs 3 scenarios: +# 1. Valid package (apache) — expect no errors +# 2. Broken package — expect multiple errors routed to correct files +# 3. Open a nested data_stream file — LSP discovers package root + +set -euo pipefail + +BINARY="${1:-./elastic-package}" + +if [[ ! -x "$BINARY" ]]; then + echo "Building elastic-package..." + go build -o "$BINARY" . +fi + +send_msg() { + local msg="$1" + local len=${#msg} + printf "Content-Length: %d\r\n\r\n%s" "$len" "$msg" +} + +run_lsp() { + local uri="$1" + local init='{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"capabilities":{}}}' + local initialized='{"jsonrpc":"2.0","method":"initialized","params":{}}' + local didopen="{\"jsonrpc\":\"2.0\",\"method\":\"textDocument/didOpen\",\"params\":{\"textDocument\":{\"uri\":\"$uri\",\"languageId\":\"yaml\",\"version\":1,\"text\":\"\"}}}" + local shutdown='{"jsonrpc":"2.0","id":2,"method":"shutdown"}' + local exit_msg='{"jsonrpc":"2.0","method":"exit"}' + + { + send_msg "$init"; sleep 0.3 + send_msg "$initialized"; sleep 0.3 + send_msg "$didopen"; sleep 3 + send_msg "$shutdown"; sleep 0.3 + send_msg "$exit_msg" + } | "$BINARY" lsp 2>/dev/null +} + +print_diags() { + python3 -c " +import sys, json + +for line in sys.stdin: + line = line.strip() + if not line: + continue + msg = json.loads(line) + uri = msg['params']['uri'] + diags = msg['params']['diagnostics'] + parts = uri.replace('file://', '').split('/') + # Show last 3-4 path components + short = '/'.join(parts[-4:]) if len(parts) > 4 else '/'.join(parts) + print(f' File: {short}') + if diags: + for d in diags: + code = d.get('code', '') + code_str = f' ({code})' if code else '' + sev = {1: 'ERROR', 2: 'WARN', 3: 'INFO', 4: 'HINT'}.get(d.get('severity', 1), '?') + print(f' [{sev}] {d[\"message\"]}{code_str}') + else: + print(' (no errors)') + print() +" +} + +extract_diags() { + sed 's/Content-Length: [0-9]*/\n---/g' | grep publishDiagnostics | print_diags +} + +# ─── Scenario 1: Valid package ──────────────────────────────────────────────── + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +INTEGRATIONS_DIR="$(cd "$REPO_ROOT/../integrations/packages" 2>/dev/null && pwd)" || true + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Scenario 1: Valid package (apache)" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +if [[ -n "$INTEGRATIONS_DIR" && -f "$INTEGRATIONS_DIR/apache/manifest.yml" ]]; then + run_lsp "file://$INTEGRATIONS_DIR/apache/manifest.yml" | extract_diags +else + echo " (skipped — ../integrations/packages/apache not found)" + echo "" +fi + +# ─── Scenario 2: Broken package ────────────────────────────────────────────── + +BROKEN_DIR=$(mktemp -d) +trap "rm -rf $BROKEN_DIR" EXIT + +mkdir -p "$BROKEN_DIR/data_stream/mystream" +cat > "$BROKEN_DIR/manifest.yml" << 'YAML' +format_version: 3.3.0 +name: INVALID-NAME +title: "" +version: 0.0.1 +type: integration +description: A broken package for testing +categories: + - security +conditions: + kibana: + version: "^8.0.0" +owner: + github: elastic/test +YAML +cat > "$BROKEN_DIR/data_stream/mystream/manifest.yml" << 'YAML' +title: My Stream +type: logs +YAML + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Scenario 2: Broken package (invalid name, missing files)" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +run_lsp "file://$BROKEN_DIR/manifest.yml" | extract_diags + +# ─── Scenario 3: Open nested file ──────────────────────────────────────────── + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Scenario 3: Open data_stream file — LSP discovers package root" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +run_lsp "file://$BROKEN_DIR/data_stream/mystream/manifest.yml" | extract_diags + +echo "Done." From 64d375b16c685761079de7f917ff060f988248be Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Tue, 24 Mar 2026 08:26:32 +0100 Subject: [PATCH 03/12] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20add=20po?= =?UTF-8?q?sition=20lookup,=20document=20store,=20and=20async=20notify=20t?= =?UTF-8?q?o=20LSP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YAML field path resolution maps validation errors to exact lines. Document store tracks open file contents for hover/completions. Notify function captured at initialize for safe async use in debouncer. Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/lsp/diagnostics.go | 5 +- internal/lsp/document.go | 163 ++++++++++++++++++++++++++++++++++ internal/lsp/document_test.go | 31 +++++++ internal/lsp/position.go | 120 +++++++++++++++++++++++++ internal/lsp/position_test.go | 63 +++++++++++++ internal/lsp/server.go | 66 +++++++++++--- 6 files changed, 433 insertions(+), 15 deletions(-) create mode 100644 internal/lsp/document.go create mode 100644 internal/lsp/document_test.go create mode 100644 internal/lsp/position.go create mode 100644 internal/lsp/position_test.go diff --git a/internal/lsp/diagnostics.go b/internal/lsp/diagnostics.go index 4ef66ee1a4..5686b999a6 100644 --- a/internal/lsp/diagnostics.go +++ b/internal/lsp/diagnostics.go @@ -67,10 +67,7 @@ func validatePackage(packageRoot string) map[string][]protocol.Diagnostic { filePath, message, code := parseError(errMsg, packageRoot) diag := protocol.Diagnostic{ - Range: protocol.Range{ - Start: protocol.Position{Line: 0, Character: 0}, - End: protocol.Position{Line: 0, Character: 0}, - }, + Range: findPosition(filePath, message), Severity: diagnosticSeverityPtr(protocol.DiagnosticSeverityError), Source: strPtr(serverName), Message: message, diff --git a/internal/lsp/document.go b/internal/lsp/document.go new file mode 100644 index 0000000000..a02f0b2fdd --- /dev/null +++ b/internal/lsp/document.go @@ -0,0 +1,163 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "os" + "strings" + "sync" + "unicode/utf16" + + protocol "github.com/tliron/glsp/protocol_3_16" +) + +// documentStore keeps the latest text for open documents so editor features can +// use unsaved content instead of falling back to the on-disk file. +type documentStore struct { + mu sync.RWMutex + text map[string]string +} + +func newDocumentStore() *documentStore { + return &documentStore{ + text: make(map[string]string), + } +} + +func (d *documentStore) Set(uri protocol.DocumentUri, text string) { + filePath, err := uriToPath(uri) + if err != nil { + return + } + + d.mu.Lock() + defer d.mu.Unlock() + d.text[filePath] = text +} + +func (d *documentStore) Update(uri protocol.DocumentUri, changes []any) { + if len(changes) == 0 { + return + } + + filePath, err := uriToPath(uri) + if err != nil { + return + } + + d.mu.Lock() + defer d.mu.Unlock() + + text := d.text[filePath] + for _, change := range changes { + switch value := change.(type) { + case protocol.TextDocumentContentChangeEventWhole: + text = value.Text + case protocol.TextDocumentContentChangeEvent: + if value.Range == nil { + text = value.Text + continue + } + text = applyTextChange(text, *value.Range, value.Text) + } + } + d.text[filePath] = text +} + +func (d *documentStore) Delete(uri protocol.DocumentUri) { + filePath, err := uriToPath(uri) + if err != nil { + return + } + + d.mu.Lock() + defer d.mu.Unlock() + delete(d.text, filePath) +} + +func (d *documentStore) Text(filePath string) (string, bool) { + d.mu.RLock() + defer d.mu.RUnlock() + text, ok := d.text[filePath] + return text, ok +} + +func (s *Server) documentText(filePath string) string { + if text, ok := s.documents.Text(filePath); ok { + return text + } + + data, err := os.ReadFile(filePath) + if err != nil { + return "" + } + return string(data) +} + +func getLineAtText(text string, lineNum int) string { + lines := splitLines(text) + if lineNum < 0 || lineNum >= len(lines) { + return "" + } + return lines[lineNum] +} + +func splitLines(text string) []string { + return strings.Split(text, "\n") +} + +func applyTextChange(text string, rng protocol.Range, replacement string) string { + runes := []rune(text) + start := positionOffset(text, rng.Start) + end := positionOffset(text, rng.End) + if start < 0 || end < start || start > len(runes) || end > len(runes) { + return text + } + + return string(runes[:start]) + replacement + string(runes[end:]) +} + +func positionOffset(text string, pos protocol.Position) int { + lines := splitLines(text) + targetLine := int(pos.Line) + if targetLine < 0 { + return 0 + } + if targetLine >= len(lines) { + return len([]rune(text)) + } + + offset := 0 + for i := 0; i < targetLine; i++ { + offset += len([]rune(lines[i])) + 1 + } + + return offset + utf16ColumnToRuneOffset(lines[targetLine], int(pos.Character)) +} + +func utf16ColumnToRuneOffset(line string, target int) int { + if target <= 0 { + return 0 + } + + offset := 0 + column := 0 + for _, r := range []rune(line) { + if column >= target { + return offset + } + column += utf16Width(r) + offset++ + } + return offset +} + +func utf16Width(r rune) int { + width := utf16.RuneLen(r) + if width < 1 { + return 1 + } + return width +} diff --git a/internal/lsp/document_test.go b/internal/lsp/document_test.go new file mode 100644 index 0000000000..0fbb85a950 --- /dev/null +++ b/internal/lsp/document_test.go @@ -0,0 +1,31 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + protocol "github.com/tliron/glsp/protocol_3_16" +) + +func TestDocumentStoreTracksLatestOpenBufferText(t *testing.T) { + store := newDocumentStore() + uri := protocol.DocumentUri("file:///tmp/manifest.yml") + + store.Set(uri, "name: old") + store.Update(uri, []any{ + protocol.TextDocumentContentChangeEventWhole{Text: "name: new"}, + }) + + text, ok := store.Text("/tmp/manifest.yml") + require.True(t, ok) + assert.Equal(t, "name: new", text) + + store.Delete(uri) + _, ok = store.Text("/tmp/manifest.yml") + assert.False(t, ok) +} diff --git a/internal/lsp/position.go b/internal/lsp/position.go new file mode 100644 index 0000000000..d15e380fca --- /dev/null +++ b/internal/lsp/position.go @@ -0,0 +1,120 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "os" + "regexp" + "strconv" + "strings" + + protocol "github.com/tliron/glsp/protocol_3_16" + yamlv3 "gopkg.in/yaml.v3" +) + +var ( + // Matches "field :" at the start of a message. + // Examples: "field processors.4:", "field title:", "field vars.1.name:" + fieldPathRe = regexp.MustCompile(`^field ([a-zA-Z0-9_.]+):`) + + // Matches "Additional property X is not allowed" + additionalPropRe = regexp.MustCompile(`Additional property (\S+) is not allowed`) +) + +// findPosition tries to locate the line in a YAML file corresponding to the +// error message. Returns a range at line 0 if the position cannot be determined. +func findPosition(filePath, message string) protocol.Range { + zero := protocol.Range{ + Start: protocol.Position{Line: 0, Character: 0}, + End: protocol.Position{Line: 0, Character: 0}, + } + + m := fieldPathRe.FindStringSubmatch(message) + if m == nil { + return zero + } + fieldPath := m[1] + + data, err := os.ReadFile(filePath) + if err != nil { + return zero + } + + var doc yamlv3.Node + if err := yamlv3.Unmarshal(data, &doc); err != nil { + return zero + } + + // The document node wraps the actual content. + if doc.Kind != yamlv3.DocumentNode || len(doc.Content) == 0 { + return zero + } + root := doc.Content[0] + + // Walk the field path (e.g. "processors.4.grokk" or "processors.4"). + segments := strings.Split(fieldPath, ".") + + // If the message mentions an additional property, append it to the path + // so we land on the exact key. + if pm := additionalPropRe.FindStringSubmatch(message); pm != nil { + segments = append(segments, pm[1]) + } + + node := walkYAMLPath(root, segments) + if node == nil { + return zero + } + + // yaml.Node lines are 1-based, LSP positions are 0-based. + line := uint32(node.Line - 1) + col := uint32(node.Column - 1) + return protocol.Range{ + Start: protocol.Position{Line: line, Character: col}, + End: protocol.Position{Line: line, Character: col + uint32(len(node.Value))}, + } +} + +// walkYAMLPath navigates a yaml.Node tree following the given path segments. +// Segments can be map keys ("processors") or array indices ("4"). +func walkYAMLPath(node *yamlv3.Node, segments []string) *yamlv3.Node { + current := node + for i, seg := range segments { + if current == nil { + return nil + } + isLast := i == len(segments)-1 + + switch current.Kind { + case yamlv3.MappingNode: + // Content alternates: key, value, key, value, ... + found := false + for j := 0; j+1 < len(current.Content); j += 2 { + if current.Content[j].Value == seg { + if isLast { + // Return the key node for its position. + return current.Content[j] + } + current = current.Content[j+1] + found = true + break + } + } + if !found { + return nil + } + + case yamlv3.SequenceNode: + idx, err := strconv.Atoi(seg) + if err != nil || idx < 0 || idx >= len(current.Content) { + return nil + } + current = current.Content[idx] + + default: + return nil + } + } + return current +} diff --git a/internal/lsp/position_test.go b/internal/lsp/position_test.go new file mode 100644 index 0000000000..940d906153 --- /dev/null +++ b/internal/lsp/position_test.go @@ -0,0 +1,63 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindPosition_FieldPath(t *testing.T) { + dir := t.TempDir() + f := filepath.Join(dir, "default.yml") + require.NoError(t, os.WriteFile(f, []byte(`--- +description: "Pipeline for parsing logs" +processors: + - set: + field: event.kind + value: event + - grok: + field: message + patterns: + - "%{COMBINEDAPACHELOG}" + - remove: + field: message + - grokk: + field: bad +`), 0644)) + + // "field processors.3:" should land on the 4th element (0-indexed: 3) + // which is the "grokk" mapping node + r := findPosition(f, "field processors.3: Additional property grokk is not allowed") + // grokk is a key inside the mapping at index 3 of the sequence + assert.Equal(t, uint32(12), r.Start.Line, "expected line 12 (0-based) for grokk key") +} + +func TestFindPosition_TopLevelField(t *testing.T) { + dir := t.TempDir() + f := filepath.Join(dir, "manifest.yml") + require.NoError(t, os.WriteFile(f, []byte(`format_version: 3.3.0 +name: my_package +title: "" +version: 0.0.1 +`), 0644)) + + r := findPosition(f, "field title: String length must be greater than or equal to 1") + assert.Equal(t, uint32(2), r.Start.Line, "expected line 2 (0-based) for title") +} + +func TestFindPosition_NoMatch(t *testing.T) { + dir := t.TempDir() + f := filepath.Join(dir, "manifest.yml") + require.NoError(t, os.WriteFile(f, []byte(`name: test +`), 0644)) + + r := findPosition(f, "some error without field path") + assert.Equal(t, uint32(0), r.Start.Line) +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go index f9f4cb969f..ba3143da94 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -17,13 +17,21 @@ const serverName = "elastic-package-lsp" var version = "0.1.0" +// notifyFunc is a function that sends a notification to the client. +type notifyFunc = glsp.NotifyFunc + // Server is the elastic-package LSP server. type Server struct { handler protocol.Handler server *server.Server debouncer *Debouncer + documents *documentStore logger commonlog.Logger + // notify is captured during initialize and used for async notifications. + notifyMu sync.Mutex + notify notifyFunc + // prevDiagFiles tracks which files had diagnostics published in the last // validation run per package root, so we can clear them when errors are fixed. prevDiagFilesMu sync.Mutex @@ -34,18 +42,22 @@ type Server struct { func NewServer() *Server { s := &Server{ debouncer: NewDebouncer(), + documents: newDocumentStore(), logger: commonlog.GetLogger(serverName), prevDiagFiles: make(map[string]map[string]struct{}), } s.handler = protocol.Handler{ Initialize: s.initialize, - Initialized: s.initialized, - Shutdown: s.shutdown, - SetTrace: s.setTrace, - TextDocumentDidOpen: s.textDocumentDidOpen, - TextDocumentDidSave: s.textDocumentDidSave, - TextDocumentDidClose: s.textDocumentDidClose, + Initialized: s.initialized, + Shutdown: s.shutdown, + SetTrace: s.setTrace, + TextDocumentDidOpen: s.textDocumentDidOpen, + TextDocumentDidChange: s.textDocumentDidChange, + TextDocumentDidSave: s.textDocumentDidSave, + TextDocumentDidClose: s.textDocumentDidClose, + TextDocumentCompletion: s.textDocumentCompletion, + TextDocumentHover: s.textDocumentHover, } s.server = server.NewServer(&s.handler, serverName, false) @@ -59,6 +71,13 @@ func (s *Server) RunStdio() error { } func (s *Server) initialize(ctx *glsp.Context, params *protocol.InitializeParams) (any, error) { + // Capture the notify function for use in async handlers (debounced validation). + s.notifyMu.Lock() + s.notify = ctx.Notify + s.notifyMu.Unlock() + + s.logger.Infof("initialize request received") + capabilities := s.handler.CreateServerCapabilities() sync := protocol.TextDocumentSyncKindFull @@ -69,6 +88,8 @@ func (s *Server) initialize(ctx *glsp.Context, params *protocol.InitializeParams IncludeText: boolPtr(false), }, } + capabilities.CompletionProvider = &protocol.CompletionOptions{} + capabilities.HoverProvider = true return protocol.InitializeResult{ Capabilities: capabilities, @@ -95,16 +116,23 @@ func (s *Server) setTrace(ctx *glsp.Context, params *protocol.SetTraceParams) er } func (s *Server) textDocumentDidOpen(ctx *glsp.Context, params *protocol.DidOpenTextDocumentParams) error { + s.documents.Set(params.TextDocument.URI, params.TextDocument.Text) s.triggerValidation(ctx, params.TextDocument.URI) return nil } +func (s *Server) textDocumentDidChange(ctx *glsp.Context, params *protocol.DidChangeTextDocumentParams) error { + s.documents.Update(params.TextDocument.URI, params.ContentChanges) + return nil +} + func (s *Server) textDocumentDidSave(ctx *glsp.Context, params *protocol.DidSaveTextDocumentParams) error { s.triggerValidation(ctx, params.TextDocument.URI) return nil } func (s *Server) textDocumentDidClose(ctx *glsp.Context, params *protocol.DidCloseTextDocumentParams) error { + s.documents.Delete(params.TextDocument.URI) return nil } @@ -115,19 +143,34 @@ func (s *Server) triggerValidation(ctx *glsp.Context, uri protocol.DocumentUri) return } + s.logger.Infof("triggerValidation for %s", filePath) + packageRoot, err := findPackageRoot(filePath) if err != nil { - // File is not inside a package — nothing to validate. + s.logger.Infof("file not inside a package: %s", filePath) return } + s.logger.Infof("found package root: %s", packageRoot) + s.debouncer.Trigger(packageRoot, func() { + s.logger.Infof("debounce fired, validating %s", packageRoot) diags := validatePackage(packageRoot) - s.publishAllDiagnostics(ctx, packageRoot, diags) + s.logger.Infof("validation returned %d file(s) with diagnostics", len(diags)) + s.publishAllDiagnostics(packageRoot, diags) }) } -func (s *Server) publishAllDiagnostics(ctx *glsp.Context, packageRoot string, diagsByFile map[string][]protocol.Diagnostic) { +func (s *Server) publishAllDiagnostics(packageRoot string, diagsByFile map[string][]protocol.Diagnostic) { + s.notifyMu.Lock() + notify := s.notify + s.notifyMu.Unlock() + + if notify == nil { + s.logger.Errorf("cannot publish diagnostics: no notify function (initialize not called?)") + return + } + s.prevDiagFilesMu.Lock() defer s.prevDiagFilesMu.Unlock() @@ -136,7 +179,7 @@ func (s *Server) publishAllDiagnostics(ctx *glsp.Context, packageRoot string, di for filePath := range prev { if _, stillHasErrors := diagsByFile[filePath]; !stillHasErrors { uri := pathToURI(filePath) - ctx.Notify(protocol.ServerTextDocumentPublishDiagnostics, protocol.PublishDiagnosticsParams{ + notify(protocol.ServerTextDocumentPublishDiagnostics, protocol.PublishDiagnosticsParams{ URI: uri, Diagnostics: []protocol.Diagnostic{}, }) @@ -148,7 +191,8 @@ func (s *Server) publishAllDiagnostics(ctx *glsp.Context, packageRoot string, di currentFiles := make(map[string]struct{}, len(diagsByFile)) for filePath, diags := range diagsByFile { uri := pathToURI(filePath) - ctx.Notify(protocol.ServerTextDocumentPublishDiagnostics, protocol.PublishDiagnosticsParams{ + s.logger.Infof("publishing %d diagnostic(s) for %s", len(diags), filePath) + notify(protocol.ServerTextDocumentPublishDiagnostics, protocol.PublishDiagnosticsParams{ URI: uri, Diagnostics: diags, }) From 85234c04f5d25652709c7e3feeee6a3eb274e955 Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Tue, 24 Mar 2026 08:26:43 +0100 Subject: [PATCH 04/12] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20field=20complet?= =?UTF-8?q?ions=20and=20hover=20docs=20to=20LSP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completions: field names from fields/*.yml in pipelines, field types in field definitions, manifest keys with context awareness. Hover: manifest key docs (nested paths resolved via indentation), field type/unit/metric_type docs, field reference info from field index. Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/lsp/completion.go | 212 ++++++++++++++ internal/lsp/fields_index.go | 92 ++++++ internal/lsp/hover.go | 326 ++++++++++++++++++++++ internal/lsp/manifest_schema.go | 403 +++++++++++++++++++++++++++ internal/lsp/manifest_schema_test.go | 53 ++++ 5 files changed, 1086 insertions(+) create mode 100644 internal/lsp/completion.go create mode 100644 internal/lsp/fields_index.go create mode 100644 internal/lsp/hover.go create mode 100644 internal/lsp/manifest_schema.go create mode 100644 internal/lsp/manifest_schema_test.go diff --git a/internal/lsp/completion.go b/internal/lsp/completion.go new file mode 100644 index 0000000000..0e2fe53a66 --- /dev/null +++ b/internal/lsp/completion.go @@ -0,0 +1,212 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/tliron/glsp" + protocol "github.com/tliron/glsp/protocol_3_16" +) + +func (s *Server) textDocumentCompletion(ctx *glsp.Context, params *protocol.CompletionParams) (any, error) { + filePath, err := uriToPath(params.TextDocument.URI) + if err != nil { + return nil, nil + } + + packageRoot, err := findPackageRoot(filePath) + if err != nil { + return nil, nil + } + + documentText := s.documentText(filePath) + line := getLineAtText(documentText, int(params.Position.Line)) + + var items []protocol.CompletionItem + + switch { + case isFieldValueContext(line): + // Completing a "field: " value in a pipeline — suggest field names. + items = s.completeFieldNames(packageRoot, filePath, line) + case isManifestFile(filePath, packageRoot): + items = completeManifestKeys(filePath, packageRoot, line, documentText) + case isFieldsDefinitionFile(filePath): + items = completeFieldTypeValues(line) + } + + if len(items) == 0 { + return nil, nil + } + return items, nil +} + +// completeFieldNames suggests dotted field names from the package's field definitions. +func (s *Server) completeFieldNames(packageRoot, filePath, line string) []protocol.CompletionItem { + // Determine which data stream we're in, if any. + ds := dataStreamFromPath(filePath, packageRoot) + var idx FieldIndex + if ds != "" { + idx = BuildFieldIndexForDataStream(packageRoot, ds) + } else { + idx = BuildFieldIndex(packageRoot) + } + + // Extract partial text after "field:" on the line. + prefix := extractFieldPrefix(line) + + var items []protocol.CompletionItem + names := make([]string, 0, len(idx)) + for name := range idx { + names = append(names, name) + } + sort.Strings(names) + + fieldKind := protocol.CompletionItemKindField + for _, name := range names { + if prefix != "" && !strings.HasPrefix(name, prefix) { + continue + } + info := idx[name] + detail := info.Type + if info.Unit != "" { + detail += " (" + info.Unit + ")" + } + doc := info.Description + item := protocol.CompletionItem{ + Label: name, + Kind: &fieldKind, + Detail: &detail, + } + if doc != "" { + item.Documentation = &protocol.MarkupContent{ + Kind: protocol.MarkupKindMarkdown, + Value: doc, + } + } + items = append(items, item) + } + return items +} + +// completeFieldTypeValues suggests valid type values when editing fields/*.yml. +func completeFieldTypeValues(line string) []protocol.CompletionItem { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "type:") { + return nil + } + + types := []string{ + "keyword", "text", "match_only_text", "wildcard", "constant_keyword", + "long", "integer", "short", "byte", "double", "float", "half_float", + "scaled_float", "unsigned_long", + "date", "date_nanos", "boolean", "binary", "ip", + "geo_point", "object", "nested", "flattened", "group", + "alias", "histogram", "aggregate_metric_double", + "integer_range", "float_range", "long_range", "double_range", + "date_range", "ip_range", + "version", "counted_keyword", "semantic_text", + } + + enumKind := protocol.CompletionItemKindEnumMember + var items []protocol.CompletionItem + for _, t := range types { + label := t + items = append(items, protocol.CompletionItem{ + Label: label, + Kind: &enumKind, + }) + } + return items +} + +// completeManifestKeys suggests top-level manifest keys based on package-spec. +func completeManifestKeys(filePath, packageRoot, line, documentText string) []protocol.CompletionItem { + trimmed := strings.TrimSpace(line) + // Only suggest at start of a line (not inside a value). + if strings.Contains(trimmed, ":") { + return nil + } + + keys := manifestTopLevelKeys(manifestSchemaKindForFile(filePath, packageRoot, documentText)) + + propKind := protocol.CompletionItemKindProperty + var items []protocol.CompletionItem + for _, k := range keys { + label := k + items = append(items, protocol.CompletionItem{ + Label: label, + Kind: &propKind, + InsertText: strPtr(k + ": "), + }) + } + return items +} + +// --- helpers --- + +func isFieldValueContext(line string) bool { + trimmed := strings.TrimSpace(line) + return strings.HasPrefix(trimmed, "field:") || + strings.HasPrefix(trimmed, "target_field:") || + strings.HasPrefix(trimmed, "source:") || + strings.HasPrefix(trimmed, "copy_to:") +} + +func extractFieldPrefix(line string) string { + for _, key := range []string{"field:", "target_field:", "source:", "copy_to:"} { + if idx := strings.Index(line, key); idx >= 0 { + return strings.TrimSpace(line[idx+len(key):]) + } + } + return "" +} + +func isManifestFile(filePath, packageRoot string) bool { + return filepath.Base(filePath) == "manifest.yml" && + strings.HasPrefix(filePath, packageRoot) +} + +func isDataStreamManifest(filePath, packageRoot string) bool { + rel, err := filepath.Rel(packageRoot, filePath) + if err != nil { + return false + } + // data_stream//manifest.yml + parts := strings.Split(rel, string(filepath.Separator)) + return len(parts) == 3 && parts[0] == "data_stream" && parts[2] == "manifest.yml" +} + +func isFieldsDefinitionFile(filePath string) bool { + return strings.Contains(filePath, string(filepath.Separator)+"fields"+string(filepath.Separator)) && + (strings.HasSuffix(filePath, ".yml") || strings.HasSuffix(filePath, ".yaml")) +} + +func dataStreamFromPath(filePath, packageRoot string) string { + rel, err := filepath.Rel(packageRoot, filePath) + if err != nil { + return "" + } + parts := strings.Split(rel, string(filepath.Separator)) + if len(parts) >= 2 && parts[0] == "data_stream" { + return parts[1] + } + return "" +} + +// formatFieldDetail returns a short detail string for a field. +func formatFieldDetail(f FieldInfo) string { + parts := []string{f.Type} + if f.Unit != "" { + parts = append(parts, fmt.Sprintf("unit: %s", f.Unit)) + } + if f.MetricType != "" { + parts = append(parts, fmt.Sprintf("metric: %s", f.MetricType)) + } + return strings.Join(parts, ", ") +} diff --git a/internal/lsp/fields_index.go b/internal/lsp/fields_index.go new file mode 100644 index 0000000000..1e497b10ae --- /dev/null +++ b/internal/lsp/fields_index.go @@ -0,0 +1,92 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "os" + "path/filepath" + "strings" + + yamlv3 "gopkg.in/yaml.v3" +) + +// FieldInfo holds metadata about a single field from fields/*.yml. +type FieldInfo struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + Description string `yaml:"description"` + Unit string `yaml:"unit,omitempty"` + MetricType string `yaml:"metric_type,omitempty"` + External string `yaml:"external,omitempty"` + Fields []FieldInfo `yaml:"fields,omitempty"` +} + +// FieldIndex is a flat map of dotted field names to their info. +type FieldIndex map[string]FieldInfo + +// BuildFieldIndex scans all fields/*.yml files under a package root and returns +// a flat index of dotted field names. +func BuildFieldIndex(packageRoot string) FieldIndex { + idx := make(FieldIndex) + + // Scan package-level fields. + collectFieldFiles(filepath.Join(packageRoot, "fields"), idx, "") + + // Scan each data stream's fields. + dsDir := filepath.Join(packageRoot, "data_stream") + entries, err := os.ReadDir(dsDir) + if err != nil { + return idx + } + for _, e := range entries { + if e.IsDir() { + collectFieldFiles(filepath.Join(dsDir, e.Name(), "fields"), idx, "") + } + } + + return idx +} + +// BuildFieldIndexForDataStream builds a field index for a specific data stream. +func BuildFieldIndexForDataStream(packageRoot, dataStream string) FieldIndex { + idx := make(FieldIndex) + collectFieldFiles(filepath.Join(packageRoot, "fields"), idx, "") + collectFieldFiles(filepath.Join(packageRoot, "data_stream", dataStream, "fields"), idx, "") + return idx +} + +func collectFieldFiles(dir string, idx FieldIndex, prefix string) { + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, e := range entries { + if e.IsDir() || (!strings.HasSuffix(e.Name(), ".yml") && !strings.HasSuffix(e.Name(), ".yaml")) { + continue + } + data, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + continue + } + var fields []FieldInfo + if err := yamlv3.Unmarshal(data, &fields); err != nil { + continue + } + flattenFields(fields, prefix, idx) + } +} + +func flattenFields(fields []FieldInfo, prefix string, idx FieldIndex) { + for _, f := range fields { + fullName := f.Name + if prefix != "" { + fullName = prefix + "." + f.Name + } + idx[fullName] = f + if f.Type == "group" && len(f.Fields) > 0 { + flattenFields(f.Fields, fullName, idx) + } + } +} diff --git a/internal/lsp/hover.go b/internal/lsp/hover.go new file mode 100644 index 0000000000..0b79be31a2 --- /dev/null +++ b/internal/lsp/hover.go @@ -0,0 +1,326 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "fmt" + "strings" + + "github.com/tliron/glsp" + protocol "github.com/tliron/glsp/protocol_3_16" +) + +func (s *Server) textDocumentHover(ctx *glsp.Context, params *protocol.HoverParams) (*protocol.Hover, error) { + filePath, err := uriToPath(params.TextDocument.URI) + if err != nil { + return nil, nil + } + + packageRoot, err := findPackageRoot(filePath) + if err != nil { + return nil, nil + } + + documentText := s.documentText(filePath) + line := getLineAtText(documentText, int(params.Position.Line)) + if line == "" { + return nil, nil + } + + // Try different hover strategies. + if md := hoverFieldReference(line, params.Position, packageRoot, filePath); md != "" { + return &protocol.Hover{ + Contents: protocol.MarkupContent{Kind: protocol.MarkupKindMarkdown, Value: md}, + }, nil + } + + if md := hoverManifestKey(line, params.Position, filePath, packageRoot, documentText); md != "" { + return &protocol.Hover{ + Contents: protocol.MarkupContent{Kind: protocol.MarkupKindMarkdown, Value: md}, + }, nil + } + + if md := hoverFieldDefinition(line, params.Position, filePath); md != "" { + return &protocol.Hover{ + Contents: protocol.MarkupContent{Kind: protocol.MarkupKindMarkdown, Value: md}, + }, nil + } + + return nil, nil +} + +// hoverFieldReference shows info when hovering over a field name in a pipeline. +func hoverFieldReference(line string, pos protocol.Position, packageRoot, filePath string) string { + // Check if cursor is on a field value (e.g. "field: message"). + fieldName := extractFieldValueAtCursor(line, pos) + if fieldName == "" { + return "" + } + + ds := dataStreamFromPath(filePath, packageRoot) + var idx FieldIndex + if ds != "" { + idx = BuildFieldIndexForDataStream(packageRoot, ds) + } else { + idx = BuildFieldIndex(packageRoot) + } + + info, ok := idx[fieldName] + if !ok { + return "" + } + + return formatFieldHover(fieldName, info) +} + +// hoverManifestKey shows documentation when hovering over a manifest key. +func hoverManifestKey(line string, pos protocol.Position, filePath, packageRoot, documentText string) string { + if !isManifestFile(filePath, packageRoot) { + return "" + } + + key := extractYAMLKey(line) + if key == "" { + return "" + } + + // Check cursor is on the key part (before the colon). + colonIdx := strings.Index(line, ":") + if colonIdx < 0 || int(pos.Character) > uint32ToInt(uint32(colonIdx)) { + return "" + } + + // Resolve full dotted path by walking up the YAML indentation. + fullPath := resolveYAMLPath(documentText, int(pos.Line)) + kind := manifestSchemaKindForFile(filePath, packageRoot, documentText) + + // Try the full path first, then progressively shorter suffixes. + for i := 0; i < len(fullPath); i++ { + candidate := strings.Join(fullPath[i:], ".") + md := manifestDoc(candidate, kind) + if md != "" { + return md + } + } + + return "" +} + +// hoverFieldDefinition shows info when hovering in a fields/*.yml file. +func hoverFieldDefinition(line string, pos protocol.Position, filePath string) string { + if !isFieldsDefinitionFile(filePath) { + return "" + } + + trimmed := strings.TrimSpace(line) + + // Hovering over a type value. + if strings.HasPrefix(trimmed, "type:") { + typeName := strings.TrimSpace(strings.TrimPrefix(trimmed, "type:")) + return fieldTypeDocs(typeName) + } + + // Hovering over a unit value. + if strings.HasPrefix(trimmed, "unit:") { + unit := strings.TrimSpace(strings.TrimPrefix(trimmed, "unit:")) + return unitDocs(unit) + } + + // Hovering over a metric_type value. + if strings.HasPrefix(trimmed, "metric_type:") { + mt := strings.TrimSpace(strings.TrimPrefix(trimmed, "metric_type:")) + return metricTypeDocs(mt) + } + + return "" +} + +// --- formatters --- + +func formatFieldHover(name string, f FieldInfo) string { + var sb strings.Builder + fmt.Fprintf(&sb, "**%s** `%s`\n\n", name, f.Type) + if f.Description != "" { + sb.WriteString(f.Description + "\n\n") + } + if f.Unit != "" { + fmt.Fprintf(&sb, "Unit: `%s`\n\n", f.Unit) + } + if f.MetricType != "" { + fmt.Fprintf(&sb, "Metric type: `%s`\n\n", f.MetricType) + } + if f.External != "" { + fmt.Fprintf(&sb, "Source: %s\n", f.External) + } + return sb.String() +} + +// --- extractors --- + +// resolveYAMLPath walks up lines from the given position to build the full +// dotted YAML key path using indentation. For example, if the cursor is on +// "input:" indented under "- " inside "streams:", it returns ["streams", "input"]. +func resolveYAMLPath(documentText string, lineNum int) []string { + lines := splitLines(documentText) + if lineNum < 0 || lineNum >= len(lines) { + return nil + } + + // Get the key and indentation of the target line. + targetKey, _, _ := yamlKeyDetails(lines[lineNum]) + if targetKey == "" { + return nil + } + + targetIndent := yamlIndent(lines[lineNum]) + path := []string{targetKey} + + // Walk upward to find parent keys at decreasing indentation. + currentIndent := targetIndent + for i := lineNum - 1; i >= 0; i-- { + line := lines[i] + indent := yamlIndent(line) + + // Skip blank lines, comments, and lines at same/deeper indent. + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + if indent >= currentIndent { + continue + } + + key, isListItem, hasInlineValue := yamlKeyDetails(line) + if key != "" { + if isListItem && hasInlineValue { + continue + } + path = append([]string{key}, path...) + currentIndent = indent + if indent == 0 { + break + } + } + } + + return path +} + +// yamlIndent returns the number of leading spaces (ignoring "- " list markers). +func yamlIndent(line string) int { + n := 0 + for _, ch := range line { + if ch == ' ' { + n++ + } else { + break + } + } + return n +} + +func extractFieldValueAtCursor(line string, pos protocol.Position) string { + for _, key := range []string{"field:", "target_field:", "source:", "copy_to:"} { + idx := strings.Index(line, key) + if idx < 0 { + continue + } + valueStart := idx + len(key) + value := strings.TrimSpace(line[valueStart:]) + // Remove quotes. + value = strings.Trim(value, `"'`) + if value != "" { + return value + } + } + return "" +} + +func extractYAMLKey(line string) string { + key, _, _ := yamlKeyDetails(line) + return key +} + +func yamlKeyDetails(line string) (key string, isListItem, hasInlineValue bool) { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") { + return "", false, false + } + if strings.HasPrefix(trimmed, "-") { + isListItem = true + trimmed = strings.TrimSpace(strings.TrimPrefix(trimmed, "-")) + } + colonIdx := strings.Index(trimmed, ":") + if colonIdx <= 0 { + return "", isListItem, false + } + key = trimmed[:colonIdx] + hasInlineValue = strings.TrimSpace(trimmed[colonIdx+1:]) != "" + return key, isListItem, hasInlineValue +} + +func fieldTypeDocs(typeName string) string { + docs := map[string]string{ + "keyword": "**keyword**\n\nExact-value string. Used for filtering, sorting, and aggregations.\n\nNot analyzed for full-text search.", + "text": "**text**\n\nFull-text searchable string. Analyzed into tokens.\n\nNot efficient for sorting or aggregations.", + "match_only_text": "**match_only_text**\n\nLike `text` but optimized for storage. Only supports match queries.\n\nNo scoring, no positions stored.", + "long": "**long**\n\n64-bit signed integer. Range: -2^63 to 2^63-1.", + "integer": "**integer**\n\n32-bit signed integer. Range: -2^31 to 2^31-1.", + "double": "**double**\n\n64-bit IEEE 754 floating point.", + "float": "**float**\n\n32-bit IEEE 754 floating point.", + "scaled_float": "**scaled_float**\n\nFloat stored as a long with a `scaling_factor`.\n\nMore storage efficient for fixed-precision decimals.", + "boolean": "**boolean**\n\nTrue/false value.", + "date": "**date**\n\nDate/time value. Supports multiple formats via `date_format`.\n\nDefault: strict_date_optional_time || epoch_millis.", + "ip": "**ip**\n\nIPv4 or IPv6 address.", + "geo_point": "**geo_point**\n\nLatitude/longitude point.", + "object": "**object**\n\nJSON object. Fields inside are flattened by default.", + "nested": "**nested**\n\nLike `object` but maintains field relationships for queries.\n\nMore expensive than `object`.", + "group": "**group**\n\nLogical grouping of sub-fields. Not an Elasticsearch type.\n\nUse `fields:` to define children.", + "flattened": "**flattened**\n\nEntire JSON object as a single field. All values treated as keywords.\n\nUseful for dynamic or unknown structures.", + "wildcard": "**wildcard**\n\nLike `keyword` but optimized for wildcard/regex queries.", + "constant_keyword": "**constant_keyword**\n\nKeyword that has the same value across all documents in the index.\n\nVery storage efficient.", + "alias": "**alias**\n\nAlternate name for an existing field. Requires `path` property.", + "histogram": "**histogram**\n\nPre-aggregated histogram values.", + "version": "**version**\n\nSemantic version string. Supports version-aware sorting.", + "unsigned_long": "**unsigned_long**\n\n64-bit unsigned integer. Range: 0 to 2^64-1.", + } + if d, ok := docs[typeName]; ok { + return d + } + return "" +} + +func unitDocs(unit string) string { + docs := map[string]string{ + "byte": "**byte** — Data size in bytes", + "percent": "**percent** — Percentage value (0-100)", + "d": "**d** — Duration in days", + "h": "**h** — Duration in hours", + "m": "**m** — Duration in minutes", + "s": "**s** — Duration in seconds", + "ms": "**ms** — Duration in milliseconds", + "micros": "**micros** — Duration in microseconds", + "nanos": "**nanos** — Duration in nanoseconds", + } + if d, ok := docs[unit]; ok { + return d + } + return "" +} + +func metricTypeDocs(mt string) string { + docs := map[string]string{ + "counter": "**counter**\n\nA cumulative metric that only increases (or resets to zero).\n\nExamples: total requests, bytes sent.", + "gauge": "**gauge**\n\nA metric that can arbitrarily go up and down.\n\nExamples: CPU usage, memory used, temperature.", + } + if d, ok := docs[mt]; ok { + return d + } + return "" +} + +func uint32ToInt(v uint32) int { + return int(v) +} diff --git a/internal/lsp/manifest_schema.go b/internal/lsp/manifest_schema.go new file mode 100644 index 0000000000..6b2765afb4 --- /dev/null +++ b/internal/lsp/manifest_schema.go @@ -0,0 +1,403 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "fmt" + "io/fs" + "path" + "sort" + "strings" + "sync" + + packagespec "github.com/elastic/package-spec/v3" + yamlv3 "gopkg.in/yaml.v3" +) + +type manifestSchemaKind string + +const ( + manifestSchemaIntegration manifestSchemaKind = "integration/manifest.spec.yml" + manifestSchemaInput manifestSchemaKind = "input/manifest.spec.yml" + manifestSchemaContent manifestSchemaKind = "content/manifest.spec.yml" + manifestSchemaDataStream manifestSchemaKind = "integration/data_stream/manifest.spec.yml" +) + +type schemaFile struct { + Spec map[string]any `yaml:"spec"` +} + +type manifestSchemaLoader struct { + fsys fs.FS + + mu sync.RWMutex + cache map[string]map[string]any +} + +var schemaLoader = newManifestSchemaLoader(packagespec.FS()) + +func newManifestSchemaLoader(fsys fs.FS) *manifestSchemaLoader { + return &manifestSchemaLoader{ + fsys: fsys, + cache: make(map[string]map[string]any), + } +} + +func manifestSchemaKindForFile(filePath, packageRoot, documentText string) manifestSchemaKind { + if isDataStreamManifest(filePath, packageRoot) { + return manifestSchemaDataStream + } + + switch packageTypeFromManifest(documentText) { + case "input": + return manifestSchemaInput + case "content": + return manifestSchemaContent + default: + return manifestSchemaIntegration + } +} + +func manifestTopLevelKeys(kind manifestSchemaKind) []string { + keys, err := schemaLoader.topLevelKeys(string(kind)) + if err != nil { + return nil + } + return keys +} + +func manifestDoc(dottedPath string, kind manifestSchemaKind) string { + doc, err := schemaLoader.docForPath(string(kind), dottedPath) + if err != nil { + return "" + } + return doc +} + +func packageTypeFromManifest(documentText string) string { + var manifest struct { + Type string `yaml:"type"` + } + if err := yamlv3.Unmarshal([]byte(documentText), &manifest); err != nil { + return "" + } + return manifest.Type +} + +func (l *manifestSchemaLoader) topLevelKeys(schemaPath string) ([]string, error) { + root, err := l.load(schemaPath) + if err != nil { + return nil, err + } + + props := asMap(root["properties"]) + if len(props) == 0 { + return nil, nil + } + + keys := make([]string, 0, len(props)) + for key := range props { + keys = append(keys, key) + } + sort.Strings(keys) + return keys, nil +} + +func (l *manifestSchemaLoader) docForPath(schemaPath, dottedPath string) (string, error) { + root, err := l.load(schemaPath) + if err != nil { + return "", err + } + + current := any(root) + currentPath := schemaPath + var required bool + segments := strings.Split(dottedPath, ".") + + for _, segment := range segments { + childPath, child, childRequired, err := l.child(currentPath, current, segment) + if err != nil { + return "", err + } + currentPath = childPath + current = child + required = childRequired + } + + _, node, err := l.normalize(currentPath, current) + if err != nil { + return "", err + } + return formatManifestDoc(dottedPath, node, required), nil +} + +func (l *manifestSchemaLoader) child(schemaPath string, node any, segment string) (string, any, bool, error) { + schemaPath, current, err := l.normalize(schemaPath, node) + if err != nil { + return "", nil, false, err + } + + if props := asMap(current["properties"]); props != nil { + if child, ok := props[segment]; ok { + return schemaPath, child, containsString(asStringSlice(current["required"]), segment), nil + } + } + + if items, ok := current["items"]; ok { + if childPath, child, required, err := l.child(schemaPath, items, segment); err == nil { + return childPath, child, required, nil + } + } + + for _, key := range []string{"allOf", "oneOf", "anyOf"} { + for _, branch := range asSlice(current[key]) { + if childPath, child, required, err := l.child(schemaPath, branch, segment); err == nil { + return childPath, child, required, nil + } + } + } + + return "", nil, false, fmt.Errorf("schema path not found: %s", segment) +} + +func (l *manifestSchemaLoader) normalize(schemaPath string, node any) (string, map[string]any, error) { + current := cloneMap(asMap(node)) + if current == nil { + return "", nil, fmt.Errorf("invalid schema node") + } + + ref := stringValue(current["$ref"]) + if ref == "" { + return schemaPath, current, nil + } + + delete(current, "$ref") + targetPath, targetNode, err := l.resolveRef(schemaPath, ref) + if err != nil { + return "", nil, err + } + + targetPath, normalizedTarget, err := l.normalize(targetPath, targetNode) + if err != nil { + return "", nil, err + } + + return targetPath, mergeMaps(normalizedTarget, current), nil +} + +func (l *manifestSchemaLoader) resolveRef(schemaPath, ref string) (string, any, error) { + filePart, fragment, _ := strings.Cut(ref, "#") + targetPath := schemaPath + if filePart != "" { + targetPath = path.Clean(path.Join(path.Dir(schemaPath), filePart)) + } + + root, err := l.load(targetPath) + if err != nil { + return "", nil, err + } + + var node any = root + if fragment == "" { + return targetPath, node, nil + } + + for _, segment := range strings.Split(strings.TrimPrefix(fragment, "/"), "/") { + next, ok := navigate(node, segment) + if !ok { + return "", nil, fmt.Errorf("invalid schema ref: %s", ref) + } + node = next + } + + return targetPath, node, nil +} + +func (l *manifestSchemaLoader) load(schemaPath string) (map[string]any, error) { + l.mu.RLock() + if cached, ok := l.cache[schemaPath]; ok { + l.mu.RUnlock() + return cached, nil + } + l.mu.RUnlock() + + data, err := fs.ReadFile(l.fsys, schemaPath) + if err != nil { + return nil, err + } + + var spec schemaFile + if err := yamlv3.Unmarshal(data, &spec); err != nil { + return nil, err + } + + l.mu.Lock() + defer l.mu.Unlock() + l.cache[schemaPath] = spec.Spec + return spec.Spec, nil +} + +func formatManifestDoc(dottedPath string, node map[string]any, required bool) string { + var parts []string + + header := fmt.Sprintf("**%s**", dottedPath) + if typeName := schemaType(node); typeName != "" { + header += fmt.Sprintf(" `%s`", typeName) + } + if required { + header += " (required)" + } + parts = append(parts, header) + + if desc := strings.TrimSpace(stringValue(node["description"])); desc != "" { + parts = append(parts, desc) + } + + if enum := schemaEnum(node); len(enum) > 0 && len(enum) <= 8 { + parts = append(parts, "Allowed values: `"+strings.Join(enum, "`, `")+"`") + } + + if defaultValue, ok := scalarValue(node["default"]); ok { + parts = append(parts, fmt.Sprintf("Default: `%s`", defaultValue)) + } + + return strings.Join(parts, "\n\n") +} + +func schemaType(node map[string]any) string { + typeValue := node["type"] + switch value := typeValue.(type) { + case string: + return value + case []any: + var parts []string + for _, item := range value { + if str, ok := item.(string); ok { + parts = append(parts, str) + } + } + return strings.Join(parts, " | ") + } + + if len(schemaEnum(node)) > 0 { + return "enum" + } + if node["properties"] != nil { + return "object" + } + if node["items"] != nil { + return "array" + } + return "" +} + +func schemaEnum(node map[string]any) []string { + values := asSlice(node["enum"]) + out := make([]string, 0, len(values)) + for _, value := range values { + if str, ok := scalarValue(value); ok { + out = append(out, str) + } + } + return out +} + +func navigate(node any, segment string) (any, bool) { + switch current := node.(type) { + case map[string]any: + next, ok := current[segment] + return next, ok + case []any: + return nil, false + default: + return nil, false + } +} + +func asMap(value any) map[string]any { + if value == nil { + return nil + } + if m, ok := value.(map[string]any); ok { + return m + } + return nil +} + +func asSlice(value any) []any { + if value == nil { + return nil + } + if list, ok := value.([]any); ok { + return list + } + return nil +} + +func asStringSlice(value any) []string { + list := asSlice(value) + out := make([]string, 0, len(list)) + for _, item := range list { + if str, ok := item.(string); ok { + out = append(out, str) + } + } + return out +} + +func cloneMap(source map[string]any) map[string]any { + if source == nil { + return nil + } + cloned := make(map[string]any, len(source)) + for key, value := range source { + cloned[key] = value + } + return cloned +} + +func mergeMaps(base, override map[string]any) map[string]any { + merged := cloneMap(base) + for key, value := range override { + merged[key] = value + } + return merged +} + +func stringValue(value any) string { + if str, ok := value.(string); ok { + return str + } + return "" +} + +func scalarValue(value any) (string, bool) { + switch v := value.(type) { + case string: + return v, true + case bool: + if v { + return "true", true + } + return "false", true + case int: + return fmt.Sprintf("%d", v), true + case int64: + return fmt.Sprintf("%d", v), true + case float64: + return fmt.Sprintf("%v", v), true + default: + return "", false + } +} + +func containsString(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} diff --git a/internal/lsp/manifest_schema_test.go b/internal/lsp/manifest_schema_test.go new file mode 100644 index 0000000000..48ecfa6d6e --- /dev/null +++ b/internal/lsp/manifest_schema_test.go @@ -0,0 +1,53 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestManifestTopLevelKeysIncludeCurrentPackageSpecFields(t *testing.T) { + keys := manifestTopLevelKeys(manifestSchemaIntegration) + + assert.Contains(t, keys, "policy_templates_behavior") + assert.Contains(t, keys, "var_groups") + assert.Contains(t, keys, "deprecated") +} + +func TestManifestDocResolvesNestedRefs(t *testing.T) { + doc := manifestDoc("policy_templates.inputs.template_paths", manifestSchemaIntegration) + + require.NotEmpty(t, doc) + assert.Contains(t, doc, "template_paths") + assert.Contains(t, doc, "array") +} + +func TestManifestDocUsesCanonicalDescriptions(t *testing.T) { + doc := manifestDoc("owner.type", manifestSchemaIntegration) + + require.NotEmpty(t, doc) + assert.Contains(t, doc, "community") + assert.Contains(t, doc, "required") +} + +func TestManifestDocCoversNestedPolicyTemplateFields(t *testing.T) { + doc := manifestDoc("policy_templates.deployment_modes", manifestSchemaIntegration) + + require.NotEmpty(t, doc) + assert.Contains(t, doc, "deployment mode") +} + +func TestResolveYAMLPathFromDocumentText(t *testing.T) { + path := resolveYAMLPath(`policy_templates: + - name: apache + inputs: + - type: logfile +`, 3) + + assert.Equal(t, []string{"policy_templates", "inputs", "type"}, path) +} From d23db5d4285eb26cbaf1c59f091111bbacf8a5bb Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Tue, 24 Mar 2026 13:37:39 +0100 Subject: [PATCH 05/12] =?UTF-8?q?=F0=9F=A9=B9=20fix:=20improve=20LSP=20err?= =?UTF-8?q?or=20parsing,=20completion=20filtering,=20and=20overlay=20FS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Better error attribution for dashboard refs and folder items. Completion prefix filtering for types and manifest keys. Overlay FS enables validation with unsaved file contents. Expose ValidateAndFilterFromFS. Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/lsp/completion.go | 30 ++-- internal/lsp/diagnostics.go | 108 +++++++++--- internal/lsp/document.go | 15 ++ internal/lsp/hover.go | 89 ++++++++-- internal/lsp/manifest_schema_test.go | 47 +++++ internal/lsp/overlay_fs.go | 118 +++++++++++++ internal/lsp/position.go | 248 +++++++++++++++++++++++---- internal/lsp/server.go | 4 +- internal/lsp/workspace.go | 2 +- internal/validation/validation.go | 25 ++- 10 files changed, 597 insertions(+), 89 deletions(-) create mode 100644 internal/lsp/overlay_fs.go diff --git a/internal/lsp/completion.go b/internal/lsp/completion.go index 0e2fe53a66..03f041791a 100644 --- a/internal/lsp/completion.go +++ b/internal/lsp/completion.go @@ -73,10 +73,7 @@ func (s *Server) completeFieldNames(packageRoot, filePath, line string) []protoc continue } info := idx[name] - detail := info.Type - if info.Unit != "" { - detail += " (" + info.Unit + ")" - } + detail := formatFieldDetail(info) doc := info.Description item := protocol.CompletionItem{ Label: name, @@ -100,6 +97,10 @@ func completeFieldTypeValues(line string) []protocol.CompletionItem { if !strings.HasPrefix(trimmed, "type:") { return nil } + prefix, _, _, ok := valueAfterKey(trimmed, "type:") + if !ok { + prefix = "" + } types := []string{ "keyword", "text", "match_only_text", "wildcard", "constant_keyword", @@ -116,6 +117,9 @@ func completeFieldTypeValues(line string) []protocol.CompletionItem { enumKind := protocol.CompletionItemKindEnumMember var items []protocol.CompletionItem for _, t := range types { + if prefix != "" && !strings.HasPrefix(t, prefix) { + continue + } label := t items = append(items, protocol.CompletionItem{ Label: label, @@ -138,6 +142,9 @@ func completeManifestKeys(filePath, packageRoot, line, documentText string) []pr propKind := protocol.CompletionItemKindProperty var items []protocol.CompletionItem for _, k := range keys { + if trimmed != "" && !strings.HasPrefix(k, trimmed) { + continue + } label := k items = append(items, protocol.CompletionItem{ Label: label, @@ -151,17 +158,18 @@ func completeManifestKeys(filePath, packageRoot, line, documentText string) []pr // --- helpers --- func isFieldValueContext(line string) bool { - trimmed := strings.TrimSpace(line) - return strings.HasPrefix(trimmed, "field:") || - strings.HasPrefix(trimmed, "target_field:") || - strings.HasPrefix(trimmed, "source:") || - strings.HasPrefix(trimmed, "copy_to:") + for _, key := range []string{"field:", "target_field:", "source:", "copy_to:"} { + if _, ok := yamlKeyValueStart(line, key); ok { + return true + } + } + return false } func extractFieldPrefix(line string) string { for _, key := range []string{"field:", "target_field:", "source:", "copy_to:"} { - if idx := strings.Index(line, key); idx >= 0 { - return strings.TrimSpace(line[idx+len(key):]) + if value, _, _, ok := valueAfterKey(line, key); ok { + return value } } return "" diff --git a/internal/lsp/diagnostics.go b/internal/lsp/diagnostics.go index 5686b999a6..0b49ec66be 100644 --- a/internal/lsp/diagnostics.go +++ b/internal/lsp/diagnostics.go @@ -5,6 +5,7 @@ package lsp import ( + "io/fs" "os" "path/filepath" "regexp" @@ -20,11 +21,20 @@ var ( // Matches: file "path/to/file" is invalid: message fileErrorRe = regexp.MustCompile(`^file "([^"]+)" is invalid: (.+)$`) + // Matches: item [name] is not allowed in folder [path] + itemInFolderErrorRe = regexp.MustCompile(`^item \[([^\]]+)\] is not allowed in folder \[([^\]]+)\]$`) + // Matches folder-based errors like: // "expecting to find [X] folder in folder [/path/to/dir]" // "expecting to find [X] file in folder [/path/to/dir]" folderErrorRe = regexp.MustCompile(`in folder \[([^\]]+)\]`) + // Matches: references found in dashboard kibana/dashboard/foo.json: id (type), ... + dashboardReferencesErrorRe = regexp.MustCompile(`^references found in dashboard ([^:]+): (.+)$`) + + // Matches: reference found in dashboard kibana/dashboard/foo.json: id (type) + dashboardReferenceErrorRe = regexp.MustCompile(`^reference found in dashboard ([^:]+): (.+)$`) + // Matches error code suffix like (SVR00001) errorCodeRe = regexp.MustCompile(`\(([A-Z]{2,3}\d{5})\)$`) ) @@ -32,9 +42,21 @@ var ( // validatePackage runs validation on the package at the given root path and // returns diagnostics grouped by absolute file path. func validatePackage(packageRoot string) map[string][]protocol.Diagnostic { + return validatePackageWith(packageRoot, os.DirFS(packageRoot), func() (error, error) { + return validation.ValidateAndFilterFromPath(packageRoot) + }) +} + +func validatePackageFS(packageRoot string, fsys fs.FS) map[string][]protocol.Diagnostic { + return validatePackageWith(packageRoot, fsys, func() (error, error) { + return validation.ValidateAndFilterFromFS(packageRoot, fsys) + }) +} + +func validatePackageWith(packageRoot string, fsys fs.FS, validate func() (error, error)) map[string][]protocol.Diagnostic { diagsByFile := make(map[string][]protocol.Diagnostic) - errs, _ := validation.ValidateAndFilterFromPath(packageRoot) + errs, _ := validate() if errs == nil { // No errors — return empty map (will clear diagnostics). // Include the manifest so we clear any previous diagnostics for it. @@ -64,20 +86,22 @@ func validatePackage(packageRoot string) map[string][]protocol.Diagnostic { } for _, errMsg := range individualErrors { - filePath, message, code := parseError(errMsg, packageRoot) + for _, expandedErrMsg := range expandDiagnosticMessages(errMsg) { + filePath, message, code := parseError(expandedErrMsg, packageRoot) + + diag := protocol.Diagnostic{ + Range: findPositionInFS(packageRoot, filePath, fsys, message), + Severity: diagnosticSeverityPtr(protocol.DiagnosticSeverityError), + Source: strPtr(serverName), + Message: message, + } - diag := protocol.Diagnostic{ - Range: findPosition(filePath, message), - Severity: diagnosticSeverityPtr(protocol.DiagnosticSeverityError), - Source: strPtr(serverName), - Message: message, - } + if code != "" { + diag.Code = &protocol.IntegerOrString{Value: code} + } - if code != "" { - diag.Code = &protocol.IntegerOrString{Value: code} + diagsByFile[filePath] = append(diagsByFile[filePath], diag) } - - diagsByFile[filePath] = append(diagsByFile[filePath], diag) } // Ensure we publish empty diagnostics for files that previously had errors @@ -87,6 +111,24 @@ func validatePackage(packageRoot string) map[string][]protocol.Diagnostic { return diagsByFile } +func expandDiagnosticMessages(errMsg string) []string { + match := dashboardReferencesErrorRe.FindStringSubmatch(errMsg) + if match == nil { + return []string{errMsg} + } + + refs := strings.Split(match[2], ", ") + if len(refs) <= 1 { + return []string{errMsg} + } + + expanded := make([]string, 0, len(refs)) + for _, ref := range refs { + expanded = append(expanded, "reference found in dashboard "+match[1]+": "+ref) + } + return expanded +} + // parseError extracts the file path, message, and error code from an error string. func parseError(errMsg string, packageRoot string) (filePath, message, code string) { // Try to extract error code. @@ -96,27 +138,45 @@ func parseError(errMsg string, packageRoot string) (filePath, message, code stri // Try the "file X is invalid: Y" pattern. if m := fileErrorRe.FindStringSubmatch(errMsg); m != nil { - rawPath := m[1] + filePath = resolveErrorPath(m[1], packageRoot) message = m[2] + return filePath, message, code + } - // The path may be absolute or relative to the package root. - if filepath.IsAbs(rawPath) { - filePath = rawPath - } else { - filePath = filepath.Join(packageRoot, rawPath) - } + // Attribute forbidden items to the exact offending file or directory path. + if m := itemInFolderErrorRe.FindStringSubmatch(errMsg); m != nil { + filePath = filepath.Join(resolveErrorPath(m[2], packageRoot), m[1]) + message = errMsg + return filePath, message, code + } + + // By-reference dashboard warnings report the file path inline instead of + // using the standard "file X is invalid" wrapper. + if m := dashboardReferenceErrorRe.FindStringSubmatch(errMsg); m != nil { + filePath = resolveErrorPath(m[1], packageRoot) + message = "reference found in dashboard: " + m[2] + return filePath, message, code + } + + if m := dashboardReferencesErrorRe.FindStringSubmatch(errMsg); m != nil { + filePath = resolveErrorPath(m[1], packageRoot) + message = "references found in dashboard: " + m[2] return filePath, message, code } // Try folder-based pattern: attribute to manifest.yml in that folder. if m := folderErrorRe.FindStringSubmatch(errMsg); m != nil { - dir := m[1] + dir := resolveErrorPath(m[1], packageRoot) candidate := filepath.Join(dir, "manifest.yml") if _, statErr := os.Stat(candidate); statErr == nil { filePath = candidate message = errMsg return filePath, message, code } + + filePath = dir + message = errMsg + return filePath, message, code } // Fallback: attribute to manifest.yml. @@ -125,6 +185,14 @@ func parseError(errMsg string, packageRoot string) (filePath, message, code stri return filePath, message, code } +func resolveErrorPath(rawPath, packageRoot string) string { + if filepath.IsAbs(rawPath) { + return filepath.Clean(rawPath) + } + + return filepath.Clean(filepath.Join(packageRoot, rawPath)) +} + // stripNumbering removes leading numbering like " 1. " from a line. func stripNumbering(s string) string { re := regexp.MustCompile(`^\s*\d+\.\s+`) diff --git a/internal/lsp/document.go b/internal/lsp/document.go index a02f0b2fdd..d2fc92a86c 100644 --- a/internal/lsp/document.go +++ b/internal/lsp/document.go @@ -84,6 +84,21 @@ func (d *documentStore) Text(filePath string) (string, bool) { return text, ok } +func (d *documentStore) Snapshot(packageRoot string) map[string]string { + d.mu.RLock() + defer d.mu.RUnlock() + + snapshot := make(map[string]string) + for filePath, text := range d.text { + relPath, ok := relativeFSPath(packageRoot, filePath) + if !ok { + continue + } + snapshot[relPath] = text + } + return snapshot +} + func (s *Server) documentText(filePath string) string { if text, ok := s.documents.Text(filePath); ok { return text diff --git a/internal/lsp/hover.go b/internal/lsp/hover.go index 0b79be31a2..aa871dc7fd 100644 --- a/internal/lsp/hover.go +++ b/internal/lsp/hover.go @@ -7,6 +7,7 @@ package lsp import ( "fmt" "strings" + "unicode" "github.com/tliron/glsp" protocol "github.com/tliron/glsp/protocol_3_16" @@ -114,23 +115,18 @@ func hoverFieldDefinition(line string, pos protocol.Position, filePath string) s return "" } - trimmed := strings.TrimSpace(line) - // Hovering over a type value. - if strings.HasPrefix(trimmed, "type:") { - typeName := strings.TrimSpace(strings.TrimPrefix(trimmed, "type:")) + if typeName := valueAfterKeyAtCursor(line, pos, "type:"); typeName != "" { return fieldTypeDocs(typeName) } // Hovering over a unit value. - if strings.HasPrefix(trimmed, "unit:") { - unit := strings.TrimSpace(strings.TrimPrefix(trimmed, "unit:")) + if unit := valueAfterKeyAtCursor(line, pos, "unit:"); unit != "" { return unitDocs(unit) } // Hovering over a metric_type value. - if strings.HasPrefix(trimmed, "metric_type:") { - mt := strings.TrimSpace(strings.TrimPrefix(trimmed, "metric_type:")) + if mt := valueAfterKeyAtCursor(line, pos, "metric_type:"); mt != "" { return metricTypeDocs(mt) } @@ -223,21 +219,80 @@ func yamlIndent(line string) int { func extractFieldValueAtCursor(line string, pos protocol.Position) string { for _, key := range []string{"field:", "target_field:", "source:", "copy_to:"} { - idx := strings.Index(line, key) - if idx < 0 { - continue - } - valueStart := idx + len(key) - value := strings.TrimSpace(line[valueStart:]) - // Remove quotes. - value = strings.Trim(value, `"'`) - if value != "" { + if value := valueAfterKeyAtCursor(line, pos, key); value != "" { return value } } return "" } +func valueAfterKeyAtCursor(line string, pos protocol.Position, key string) string { + value, start, end, ok := valueAfterKey(line, key) + if !ok { + return "" + } + + cursor := utf16ColumnToRuneOffset(line, int(pos.Character)) + if cursor < start || cursor >= end { + return "" + } + + return value +} + +func valueAfterKey(line string, key string) (string, int, int, bool) { + start, ok := yamlKeyValueStart(line, key) + if !ok { + return "", 0, 0, false + } + + runes := []rune(line) + for start < len(runes) && unicode.IsSpace(runes[start]) { + start++ + } + + if start >= len(runes) { + return "", 0, 0, false + } + + quote := rune(0) + if runes[start] == '"' || runes[start] == '\'' { + quote = runes[start] + start++ + } + + end := start + if quote != 0 { + for end < len(runes) && runes[end] != quote { + end++ + } + } else { + for end < len(runes) && !unicode.IsSpace(runes[end]) && runes[end] != '#' { + end++ + } + } + + if end <= start { + return "", 0, 0, false + } + + return string(runes[start:end]), start, end, true +} + +func yamlKeyValueStart(line string, key string) (int, bool) { + trimmed := strings.TrimLeftFunc(line, unicode.IsSpace) + leadingRunes := len([]rune(line)) - len([]rune(trimmed)) + + switch { + case strings.HasPrefix(trimmed, key): + return leadingRunes + len([]rune(key)), true + case strings.HasPrefix(trimmed, "- "+key): + return leadingRunes + len([]rune("- "+key)), true + default: + return 0, false + } +} + func extractYAMLKey(line string) string { key, _, _ := yamlKeyDetails(line) return key diff --git a/internal/lsp/manifest_schema_test.go b/internal/lsp/manifest_schema_test.go index 48ecfa6d6e..fbd345225d 100644 --- a/internal/lsp/manifest_schema_test.go +++ b/internal/lsp/manifest_schema_test.go @@ -5,6 +5,7 @@ package lsp import ( + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -51,3 +52,49 @@ func TestResolveYAMLPathFromDocumentText(t *testing.T) { assert.Equal(t, []string{"policy_templates", "inputs", "type"}, path) } + +func TestManifestSchemaKindForFileUsesPackageTypeAndDataStreamPath(t *testing.T) { + packageRoot := "/tmp/pkg" + + assert.Equal(t, manifestSchemaInput, manifestSchemaKindForFile( + filepath.Join(packageRoot, "manifest.yml"), + packageRoot, + "type: input\n", + )) + assert.Equal(t, manifestSchemaContent, manifestSchemaKindForFile( + filepath.Join(packageRoot, "manifest.yml"), + packageRoot, + "type: content\n", + )) + assert.Equal(t, manifestSchemaIntegration, manifestSchemaKindForFile( + filepath.Join(packageRoot, "manifest.yml"), + packageRoot, + "type: integration\n", + )) + assert.Equal(t, manifestSchemaDataStream, manifestSchemaKindForFile( + filepath.Join(packageRoot, "data_stream", "access", "manifest.yml"), + packageRoot, + "type: input\n", + )) +} + +func TestManifestSchemaHelperFunctions(t *testing.T) { + assert.Equal(t, "input", packageTypeFromManifest("type: input\n")) + assert.Equal(t, "", packageTypeFromManifest(":\n")) + assert.Equal(t, "string | null", schemaType(map[string]any{"type": []any{"string", "null"}})) + assert.Equal(t, "enum", schemaType(map[string]any{"enum": []any{"a", "b"}})) + assert.Equal(t, "object", schemaType(map[string]any{"properties": map[string]any{"name": "x"}})) + assert.Equal(t, "array", schemaType(map[string]any{"items": map[string]any{"type": "string"}})) + assert.Equal(t, "", schemaType(map[string]any{})) + + value, ok := scalarValue(true) + require.True(t, ok) + assert.Equal(t, "true", value) + + value, ok = scalarValue(1.5) + require.True(t, ok) + assert.Equal(t, "1.5", value) + + _, ok = scalarValue(map[string]any{"bad": "value"}) + assert.False(t, ok) +} diff --git a/internal/lsp/overlay_fs.go b/internal/lsp/overlay_fs.go new file mode 100644 index 0000000000..82943da322 --- /dev/null +++ b/internal/lsp/overlay_fs.go @@ -0,0 +1,118 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +// overlayFS serves package files from disk while letting open editor buffers +// override specific files with unsaved content. +type overlayFS struct { + base fs.FS + overrides map[string]string +} + +func newOverlayFS(packageRoot string, overrides map[string]string) fs.FS { + return &overlayFS{ + base: os.DirFS(packageRoot), + overrides: overrides, + } +} + +func (o *overlayFS) Open(name string) (fs.File, error) { + normalized, err := normalizeFSPath(name) + if err != nil { + return nil, err + } + + if text, ok := o.overrides[normalized]; ok { + return &virtualFile{ + Reader: strings.NewReader(text), + info: virtualFileInfo{ + name: path.Base(normalized), + size: int64(len(text)), + }, + }, nil + } + + return o.base.Open(normalized) +} + +type virtualFile struct { + *strings.Reader + info virtualFileInfo +} + +func (f *virtualFile) Stat() (fs.FileInfo, error) { + return f.info, nil +} + +func (f *virtualFile) Close() error { + return nil +} + +type virtualFileInfo struct { + name string + size int64 +} + +func (i virtualFileInfo) Name() string { + return i.name +} + +func (i virtualFileInfo) Size() int64 { + return i.size +} + +func (i virtualFileInfo) Mode() fs.FileMode { + return 0o444 +} + +func (i virtualFileInfo) ModTime() time.Time { + return time.Time{} +} + +func (i virtualFileInfo) IsDir() bool { + return false +} + +func (i virtualFileInfo) Sys() any { + return nil +} + +func relativeFSPath(packageRoot, filePath string) (string, bool) { + if packageRoot == "" { + return "", false + } + + relPath, err := filepath.Rel(packageRoot, filePath) + if err != nil { + return "", false + } + relPath = filepath.ToSlash(relPath) + if relPath == "." || relPath == "" || relPath == ".." || strings.HasPrefix(relPath, "../") { + return "", false + } + return relPath, true +} + +func normalizeFSPath(name string) (string, error) { + switch name { + case "", ".": + return ".", nil + } + + normalized := path.Clean(strings.TrimPrefix(name, "./")) + if !fs.ValidPath(normalized) { + return "", &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} + } + return normalized, nil +} diff --git a/internal/lsp/position.go b/internal/lsp/position.go index d15e380fca..f84b4e6e40 100644 --- a/internal/lsp/position.go +++ b/internal/lsp/position.go @@ -5,6 +5,7 @@ package lsp import ( + "io/fs" "os" "regexp" "strconv" @@ -16,29 +17,56 @@ import ( var ( // Matches "field :" at the start of a message. - // Examples: "field processors.4:", "field title:", "field vars.1.name:" - fieldPathRe = regexp.MustCompile(`^field ([a-zA-Z0-9_.]+):`) + // Examples: "field processors.4:", "field title:", "field (root):" + fieldPathRe = regexp.MustCompile(`^field ([^:]+):`) // Matches "Additional property X is not allowed" additionalPropRe = regexp.MustCompile(`Additional property (\S+) is not allowed`) + + // Matches messages like "set processor at line 3 missing required tag" + lineNumberRe = regexp.MustCompile(`\bat line (\d+)\b`) + + // Matches "dangling reference found: object-id (search)" + danglingReferenceRe = regexp.MustCompile(`^dangling reference found: ([^ ]+) \(([^)]+)\)`) + + // Matches "reference found in dashboard: object-id (search)" + dashboardReferenceMessageRe = regexp.MustCompile(`^references? found in dashboard: ([^ ]+) \(([^)]+)\)`) + + // Matches legacy visualization messages on dashboard files. + dashboardLegacyVisualizationRe = regexp.MustCompile(`^"([^"]+)" contains legacy visualization: "([^"]*)" \(([^,]+), ([^)]+)\)$`) + + // Matches legacy visualization messages on visualization files. + legacyVisualizationRe = regexp.MustCompile(`^found legacy visualization "([^"]*)" \(([^,]+), ([^)]+)\)$`) ) // findPosition tries to locate the line in a YAML file corresponding to the // error message. Returns a range at line 0 if the position cannot be determined. func findPosition(filePath, message string) protocol.Range { + return findPositionInFS("", filePath, nil, message) +} + +func findPositionInFS(packageRoot, filePath string, fsys fs.FS, message string) protocol.Range { zero := protocol.Range{ Start: protocol.Position{Line: 0, Character: 0}, End: protocol.Position{Line: 0, Character: 0}, } - m := fieldPathRe.FindStringSubmatch(message) - if m == nil { + data, err := readDiagnosticFile(packageRoot, filePath, fsys) + if err != nil { return zero } - fieldPath := m[1] + text := string(data) - data, err := os.ReadFile(filePath) - if err != nil { + if location, ok := findLineNumberPosition(text, message); ok { + return location + } + + if location, ok := findJSONPosition(filePath, text, message); ok { + return location + } + + fieldPath, ok := extractFieldPath(message) + if !ok { return zero } @@ -53,16 +81,7 @@ func findPosition(filePath, message string) protocol.Range { } root := doc.Content[0] - // Walk the field path (e.g. "processors.4.grokk" or "processors.4"). - segments := strings.Split(fieldPath, ".") - - // If the message mentions an additional property, append it to the path - // so we land on the exact key. - if pm := additionalPropRe.FindStringSubmatch(message); pm != nil { - segments = append(segments, pm[1]) - } - - node := walkYAMLPath(root, segments) + node := resolveDiagnosticNode(root, fieldPath, message) if node == nil { return zero } @@ -76,34 +95,182 @@ func findPosition(filePath, message string) protocol.Range { } } +func extractFieldPath(message string) (string, bool) { + match := fieldPathRe.FindStringSubmatch(message) + if match == nil { + return "", false + } + return strings.TrimSpace(match[1]), true +} + +func findLineNumberPosition(text, message string) (protocol.Range, bool) { + match := lineNumberRe.FindStringSubmatch(message) + if match == nil { + return protocol.Range{}, false + } + + lineNumber, err := strconv.Atoi(match[1]) + if err != nil || lineNumber <= 0 { + return protocol.Range{}, false + } + + return rangeForLine(text, lineNumber-1), true +} + +func findJSONPosition(filePath, text, message string) (protocol.Range, bool) { + if !strings.HasSuffix(filePath, ".json") { + return protocol.Range{}, false + } + + if match := danglingReferenceRe.FindStringSubmatch(message); match != nil { + return findJSONPropertyValueRange(text, "id", match[1]) + } + + if match := dashboardReferenceMessageRe.FindStringSubmatch(message); match != nil { + return findJSONPropertyValueRange(text, "id", match[1]) + } + + if match := dashboardLegacyVisualizationRe.FindStringSubmatch(message); match != nil { + if match[2] != "" { + if location, ok := findJSONPropertyValueRange(text, "title", match[2]); ok { + return location, true + } + } + if location, ok := findJSONPropertyValueRange(text, "title", match[1]); ok { + return location, true + } + return findJSONPropertyValueRange(text, "type", match[3]) + } + + if match := legacyVisualizationRe.FindStringSubmatch(message); match != nil { + if match[1] != "" { + if location, ok := findJSONPropertyValueRange(text, "title", match[1]); ok { + return location, true + } + } + return findJSONPropertyValueRange(text, "type", match[2]) + } + + return protocol.Range{}, false +} + +func resolveDiagnosticNode(root *yamlv3.Node, fieldPath, message string) *yamlv3.Node { + current := root + if fieldPath != "" && fieldPath != "(root)" { + if pm := additionalPropRe.FindStringSubmatch(message); pm != nil { + current = walkYAMLValuePath(root, strings.Split(fieldPath, ".")) + } else { + current = walkYAMLPath(root, strings.Split(fieldPath, ".")) + } + if current == nil { + return nil + } + } + + if pm := additionalPropRe.FindStringSubmatch(message); pm != nil { + return walkYAMLPath(current, strings.Split(pm[1], ".")) + } + + return current +} + +func readDiagnosticFile(packageRoot, filePath string, fsys fs.FS) ([]byte, error) { + if fsys != nil { + if relPath, ok := relativeFSPath(packageRoot, filePath); ok { + data, err := fs.ReadFile(fsys, relPath) + if err == nil { + return data, nil + } + } + } + + return os.ReadFile(filePath) +} + +func findJSONPropertyValueRange(text, key, value string) (protocol.Range, bool) { + lines := splitLines(text) + quotedValue := strconv.Quote(value) + + for lineIndex, line := range lines { + if !strings.Contains(line, quotedValue) { + continue + } + if key != "" && !strings.Contains(line, `"`+key+`"`) { + continue + } + + start := strings.Index(line, quotedValue) + if start < 0 { + continue + } + + return protocol.Range{ + Start: protocol.Position{Line: uint32(lineIndex), Character: uint32(start + 1)}, + End: protocol.Position{Line: uint32(lineIndex), Character: uint32(start + 1 + len(value))}, + }, true + } + + return protocol.Range{}, false +} + +func rangeForLine(text string, lineNumber int) protocol.Range { + lines := splitLines(text) + if lineNumber < 0 || lineNumber >= len(lines) { + return protocol.Range{ + Start: protocol.Position{Line: 0, Character: 0}, + End: protocol.Position{Line: 0, Character: 0}, + } + } + + line := lines[lineNumber] + start := len(line) - len(strings.TrimLeft(line, " \t")) + end := len(line) + if end < start { + end = start + } + + return protocol.Range{ + Start: protocol.Position{Line: uint32(lineNumber), Character: uint32(start)}, + End: protocol.Position{Line: uint32(lineNumber), Character: uint32(end)}, + } +} + // walkYAMLPath navigates a yaml.Node tree following the given path segments. // Segments can be map keys ("processors") or array indices ("4"). func walkYAMLPath(node *yamlv3.Node, segments []string) *yamlv3.Node { + return walkYAMLPathMode(node, segments, false) +} + +func walkYAMLValuePath(node *yamlv3.Node, segments []string) *yamlv3.Node { + return walkYAMLPathMode(node, segments, true) +} + +func walkYAMLPathMode(node *yamlv3.Node, segments []string, returnValue bool) *yamlv3.Node { current := node - for i, seg := range segments { + if len(segments) == 0 { + return current + } + + for i := 0; i < len(segments); i++ { if current == nil { return nil } - isLast := i == len(segments)-1 + seg := segments[i] switch current.Kind { case yamlv3.MappingNode: - // Content alternates: key, value, key, value, ... - found := false - for j := 0; j+1 < len(current.Content); j += 2 { - if current.Content[j].Value == seg { - if isLast { - // Return the key node for its position. - return current.Content[j] - } - current = current.Content[j+1] - found = true - break - } - } - if !found { + keyNode, valueNode, nextIndex := findMappingPathMatch(current, segments, i) + if keyNode == nil { return nil } + if nextIndex == len(segments) { + if returnValue { + return valueNode + } + return keyNode + } + current = valueNode + i = nextIndex - 1 case yamlv3.SequenceNode: idx, err := strconv.Atoi(seg) @@ -111,6 +278,9 @@ func walkYAMLPath(node *yamlv3.Node, segments []string) *yamlv3.Node { return nil } current = current.Content[idx] + if i == len(segments)-1 { + return current + } default: return nil @@ -118,3 +288,15 @@ func walkYAMLPath(node *yamlv3.Node, segments []string) *yamlv3.Node { } return current } + +func findMappingPathMatch(node *yamlv3.Node, segments []string, start int) (*yamlv3.Node, *yamlv3.Node, int) { + for end := len(segments); end > start; end-- { + candidate := strings.Join(segments[start:end], ".") + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == candidate { + return node.Content[i], node.Content[i+1], end + } + } + } + return nil, nil, 0 +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go index ba3143da94..832ca21e0a 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -123,6 +123,7 @@ func (s *Server) textDocumentDidOpen(ctx *glsp.Context, params *protocol.DidOpen func (s *Server) textDocumentDidChange(ctx *glsp.Context, params *protocol.DidChangeTextDocumentParams) error { s.documents.Update(params.TextDocument.URI, params.ContentChanges) + s.triggerValidation(ctx, params.TextDocument.URI) return nil } @@ -133,6 +134,7 @@ func (s *Server) textDocumentDidSave(ctx *glsp.Context, params *protocol.DidSave func (s *Server) textDocumentDidClose(ctx *glsp.Context, params *protocol.DidCloseTextDocumentParams) error { s.documents.Delete(params.TextDocument.URI) + s.triggerValidation(ctx, params.TextDocument.URI) return nil } @@ -155,7 +157,7 @@ func (s *Server) triggerValidation(ctx *glsp.Context, uri protocol.DocumentUri) s.debouncer.Trigger(packageRoot, func() { s.logger.Infof("debounce fired, validating %s", packageRoot) - diags := validatePackage(packageRoot) + diags := validatePackageFS(packageRoot, newOverlayFS(packageRoot, s.documents.Snapshot(packageRoot))) s.logger.Infof("validation returned %d file(s) with diagnostics", len(diags)) s.publishAllDiagnostics(packageRoot, diags) }) diff --git a/internal/lsp/workspace.go b/internal/lsp/workspace.go index c79f5a0014..efb3f1b790 100644 --- a/internal/lsp/workspace.go +++ b/internal/lsp/workspace.go @@ -47,5 +47,5 @@ func pathToURI(path string) string { if !strings.HasPrefix(path, "/") { path = "/" + path } - return "file://" + path + return (&url.URL{Scheme: "file", Path: path}).String() } diff --git a/internal/validation/validation.go b/internal/validation/validation.go index b58083e18b..73d75d6f9b 100644 --- a/internal/validation/validation.go +++ b/internal/validation/validation.go @@ -23,18 +23,23 @@ func ValidateFromZip(packagePath string) error { return validator.ValidateFromZip(packagePath) } -func ValidateAndFilterFromPath(packageRoot string) (error, error) { - allErrors := validator.ValidateFromPath(packageRoot) +func ValidateAndFilterFromFS(packageRoot string, fsys fs.FS) (error, error) { + allErrors := validator.ValidateFromFS(packageRoot, fsys) if allErrors == nil { return nil, nil } + return filterValidationErrors(allErrors, fsys) +} + +func ValidateAndFilterFromPath(packageRoot string) (error, error) { fsys := os.DirFS(packageRoot) - result, err := filterErrors(allErrors, fsys) - if err != nil { - return err, nil + allErrors := validator.ValidateFromPath(packageRoot) + if allErrors == nil { + return nil, nil } - return result.Processed, result.Removed + + return filterValidationErrors(allErrors, fsys) } func ValidateAndFilterFromZip(zipPackagePath string) (error, error) { @@ -77,6 +82,14 @@ func fsFromPackageZip(fsys fs.FS) (fs.FS, error) { return subDir, nil } +func filterValidationErrors(allErrors error, fsys fs.FS) (error, error) { + result, err := filterErrors(allErrors, fsys) + if err != nil { + return err, nil + } + return result.Processed, result.Removed +} + func filterErrors(allErrors error, fsys fs.FS) (specerrors.FilterResult, error) { errs, ok := allErrors.(specerrors.ValidationErrors) if !ok { From 758c73c18209b197713f8077e1efeec04378090c Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Tue, 24 Mar 2026 13:37:49 +0100 Subject: [PATCH 06/12] =?UTF-8?q?=E2=9C=85=20test:=20add=20comprehensive?= =?UTF-8?q?=20LSP=20test=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests for completions, hover, field index, workspace, overlay FS, document store, server lifecycle, and live diagnostics with real package validation. Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/lsp/completion_test.go | 120 +++++++++++++++ internal/lsp/diagnostics_test.go | 33 +++++ internal/lsp/document_test.go | 46 ++++++ internal/lsp/fields_index_test.go | 32 ++++ internal/lsp/hover_test.go | 70 +++++++++ internal/lsp/live_diagnostics_test.go | 204 ++++++++++++++++++++++++++ internal/lsp/overlay_fs_test.go | 38 +++++ internal/lsp/position_test.go | 115 ++++++++++++++- internal/lsp/server_test.go | 64 ++++++++ internal/lsp/workspace_test.go | 29 ++++ 10 files changed, 747 insertions(+), 4 deletions(-) create mode 100644 internal/lsp/completion_test.go create mode 100644 internal/lsp/fields_index_test.go create mode 100644 internal/lsp/hover_test.go create mode 100644 internal/lsp/live_diagnostics_test.go create mode 100644 internal/lsp/overlay_fs_test.go create mode 100644 internal/lsp/server_test.go create mode 100644 internal/lsp/workspace_test.go diff --git a/internal/lsp/completion_test.go b/internal/lsp/completion_test.go new file mode 100644 index 0000000000..106f2be32b --- /dev/null +++ b/internal/lsp/completion_test.go @@ -0,0 +1,120 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + protocol "github.com/tliron/glsp/protocol_3_16" +) + +func TestCompletionHelpers(t *testing.T) { + packageRoot := "/tmp/apache" + manifestPath := filepath.Join(packageRoot, "manifest.yml") + dataStreamManifestPath := filepath.Join(packageRoot, "data_stream", "access", "manifest.yml") + fieldsPath := filepath.Join(packageRoot, "data_stream", "access", "fields", "fields.yml") + + assert.True(t, isFieldValueContext(" field: apache.access.ssl.protocol")) + assert.True(t, isFieldValueContext(" - field: apache.access.ssl.protocol")) + assert.False(t, isFieldValueContext("title: Apache")) + assert.Equal(t, "apache.access.ssl.", extractFieldPrefix("field: apache.access.ssl.")) + assert.Equal(t, "apache.access.ssl.", extractFieldPrefix(`field: "apache.access.ssl.`)) + assert.True(t, isManifestFile(manifestPath, packageRoot)) + assert.True(t, isDataStreamManifest(dataStreamManifestPath, packageRoot)) + assert.True(t, isFieldsDefinitionFile(fieldsPath)) + assert.Equal(t, "access", dataStreamFromPath(fieldsPath, packageRoot)) + assert.Equal(t, "long, unit: byte, metric: counter", formatFieldDetail(FieldInfo{ + Type: "long", + Unit: "byte", + MetricType: "counter", + })) +} + +func TestCompleteFieldNamesAndManifestKeys(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + server := NewServer() + + accessManifestPath := filepath.Join(packageRoot, "data_stream", "access", "manifest.yml") + fieldItems := server.completeFieldNames(packageRoot, accessManifestPath, "field: apache.access.ssl.") + require.NotEmpty(t, fieldItems) + assert.NotNil(t, findCompletionItem(fieldItems, "apache.access.ssl.protocol")) + assert.NotNil(t, findCompletionItem(fieldItems, "apache.access.ssl.cipher")) + assert.NotNil(t, findCompletionItem(server.completeFieldNames(packageRoot, accessManifestPath, "- field: apache.access.ssl."), "apache.access.ssl.protocol")) + + statusManifestPath := filepath.Join(packageRoot, "data_stream", "status", "manifest.yml") + statusFieldItems := server.completeFieldNames(packageRoot, statusManifestPath, `field: "apache.status.total_`) + statusItem := findCompletionItem(statusFieldItems, "apache.status.total_bytes") + require.NotNil(t, statusItem) + require.NotNil(t, statusItem.Detail) + assert.Equal(t, "long, unit: byte, metric: counter", *statusItem.Detail) + + manifestPath := filepath.Join(packageRoot, "manifest.yml") + manifestItems := completeManifestKeys(manifestPath, packageRoot, "pol", readTestFile(t, manifestPath)) + require.NotEmpty(t, manifestItems) + item := findCompletionItem(manifestItems, "policy_templates") + require.NotNil(t, item) + require.NotNil(t, item.InsertText) + assert.Equal(t, "policy_templates: ", *item.InsertText) + assert.Nil(t, findCompletionItem(manifestItems, "title")) + + assert.Nil(t, completeManifestKeys(manifestPath, packageRoot, "title: Apache", readTestFile(t, manifestPath))) +} + +func TestCompleteFieldTypeValuesSuggestsKnownTypes(t *testing.T) { + items := completeFieldTypeValues("type: k") + + require.NotEmpty(t, items) + assert.NotNil(t, findCompletionItem(items, "keyword")) + assert.Nil(t, findCompletionItem(items, "boolean")) + assert.NotNil(t, findCompletionItem(completeFieldTypeValues(`type: "ke`), "keyword")) + assert.Nil(t, completeFieldTypeValues("name: apache")) +} + +func TestTextDocumentCompletionUsesOpenBufferContent(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + server := NewServer() + + manifestPath := filepath.Join(packageRoot, "manifest.yml") + manifestURI := protocol.DocumentUri(pathToURI(manifestPath)) + server.documents.Set(manifestURI, "title") + + result, err := server.textDocumentCompletion(nil, &protocol.CompletionParams{ + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ + TextDocument: protocol.TextDocumentIdentifier{URI: manifestURI}, + Position: protocol.Position{Line: 0, Character: 3}, + }, + }) + require.NoError(t, err) + manifestItems, ok := result.([]protocol.CompletionItem) + require.True(t, ok) + assert.NotNil(t, findCompletionItem(manifestItems, "title")) + + fieldsPath := filepath.Join(packageRoot, "data_stream", "access", "fields", "tmp.yml") + fieldsURI := protocol.DocumentUri(pathToURI(fieldsPath)) + server.documents.Set(fieldsURI, "type: ") + + result, err = server.textDocumentCompletion(nil, &protocol.CompletionParams{ + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ + TextDocument: protocol.TextDocumentIdentifier{URI: fieldsURI}, + Position: protocol.Position{Line: 0, Character: 6}, + }, + }) + require.NoError(t, err) + typeItems, ok := result.([]protocol.CompletionItem) + require.True(t, ok) + assert.NotNil(t, findCompletionItem(typeItems, "keyword")) +} + +func findCompletionItem(items []protocol.CompletionItem, label string) *protocol.CompletionItem { + for i := range items { + if items[i].Label == label { + return &items[i] + } + } + return nil +} diff --git a/internal/lsp/diagnostics_test.go b/internal/lsp/diagnostics_test.go index c2fab2cef4..5919d5734d 100644 --- a/internal/lsp/diagnostics_test.go +++ b/internal/lsp/diagnostics_test.go @@ -63,6 +63,39 @@ func TestParseError_NoFilePattern(t *testing.T) { assert.Equal(t, "", code) } +func TestParseError_ItemInFolder(t *testing.T) { + packageRoot := "/home/user/packages/apache" + + filePath, message, code := parseError( + `item [routing_rules.yml] is not allowed in folder [data_stream/rules]`, + packageRoot, + ) + + assert.Equal(t, "/home/user/packages/apache/data_stream/rules/routing_rules.yml", filePath) + assert.Equal(t, `item [routing_rules.yml] is not allowed in folder [data_stream/rules]`, message) + assert.Equal(t, "", code) +} + +func TestParseError_DashboardReferenceWarning(t *testing.T) { + packageRoot := "/home/user/packages/apache" + + filePath, message, code := parseError( + `reference found in dashboard kibana/dashboard/example.json: missing-ref (search) (SVR00004)`, + packageRoot, + ) + + assert.Equal(t, "/home/user/packages/apache/kibana/dashboard/example.json", filePath) + assert.Equal(t, "reference found in dashboard: missing-ref (search) (SVR00004)", message) + assert.Equal(t, "SVR00004", code) +} + +func TestExpandDiagnosticMessagesSplitsDashboardReferences(t *testing.T) { + assert.Equal(t, []string{ + `reference found in dashboard kibana/dashboard/example.json: first-ref (search)`, + `reference found in dashboard kibana/dashboard/example.json: second-ref (lens)`, + }, expandDiagnosticMessages(`references found in dashboard kibana/dashboard/example.json: first-ref (search), second-ref (lens)`)) +} + func TestStripNumbering(t *testing.T) { assert.Equal(t, "hello world", stripNumbering(" 1. hello world")) assert.Equal(t, "error msg", stripNumbering(" 12. error msg")) diff --git a/internal/lsp/document_test.go b/internal/lsp/document_test.go index 0fbb85a950..a5830ab7e7 100644 --- a/internal/lsp/document_test.go +++ b/internal/lsp/document_test.go @@ -5,6 +5,8 @@ package lsp import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -29,3 +31,47 @@ func TestDocumentStoreTracksLatestOpenBufferText(t *testing.T) { _, ok = store.Text("/tmp/manifest.yml") assert.False(t, ok) } + +func TestDocumentStoreAppliesIncrementalChanges(t *testing.T) { + store := newDocumentStore() + uri := protocol.DocumentUri("file:///tmp/manifest.yml") + + store.Set(uri, "title: old") + store.Update(uri, []any{ + protocol.TextDocumentContentChangeEvent{ + Range: &protocol.Range{ + Start: protocol.Position{Line: 0, Character: 7}, + End: protocol.Position{Line: 0, Character: 10}, + }, + Text: "new", + }, + }) + + text, ok := store.Text("/tmp/manifest.yml") + require.True(t, ok) + assert.Equal(t, "title: new", text) +} + +func TestDocumentTextPrefersOpenBufferAndFallsBackToDisk(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "manifest.yml") + require.NoError(t, os.WriteFile(filePath, []byte("title: disk"), 0o644)) + + server := NewServer() + uri := protocol.DocumentUri(pathToURI(filePath)) + + assert.Equal(t, "title: disk", server.documentText(filePath)) + + server.documents.Set(uri, "title: buffer") + assert.Equal(t, "title: buffer", server.documentText(filePath)) +} + +func TestTextOffsetHelpersHandleUTF16Columns(t *testing.T) { + assert.Equal(t, 2, utf16ColumnToRuneOffset("a😀b", 3)) + assert.Equal(t, 4, positionOffset("x\na😀b", protocol.Position{Line: 1, Character: 3})) + assert.Equal(t, "aXb", applyTextChange("a😀b", protocol.Range{ + Start: protocol.Position{Line: 0, Character: 1}, + End: protocol.Position{Line: 0, Character: 3}, + }, "X")) + assert.Equal(t, 2, utf16Width('😀')) +} diff --git a/internal/lsp/fields_index_test.go b/internal/lsp/fields_index_test.go new file mode 100644 index 0000000000..8bf091fe07 --- /dev/null +++ b/internal/lsp/fields_index_test.go @@ -0,0 +1,32 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildFieldIndexIncludesNestedFieldsAcrossDataStreams(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + + idx := BuildFieldIndex(packageRoot) + + assert.Contains(t, idx, "apache.access.ssl.protocol") + assert.Contains(t, idx, "apache.error.module") + assert.Contains(t, idx, "apache.status.total_bytes") + assert.Equal(t, "keyword", idx["apache.access.ssl.protocol"].Type) + assert.Equal(t, "byte", idx["apache.status.total_bytes"].Unit) +} + +func TestBuildFieldIndexForDataStreamScopesResults(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + + idx := BuildFieldIndexForDataStream(packageRoot, "status") + + assert.Contains(t, idx, "apache.status.total_bytes") + assert.NotContains(t, idx, "apache.access.ssl.protocol") +} diff --git a/internal/lsp/hover_test.go b/internal/lsp/hover_test.go new file mode 100644 index 0000000000..eae6fa2b9b --- /dev/null +++ b/internal/lsp/hover_test.go @@ -0,0 +1,70 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + protocol "github.com/tliron/glsp/protocol_3_16" +) + +func TestHoverFieldReferenceAndFormatting(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + filePath := filepath.Join(packageRoot, "data_stream", "access", "manifest.yml") + line := `field: "apache.access.ssl.protocol"` + + md := hoverFieldReference(line, protocol.Position{Line: 0, Character: 12}, packageRoot, filePath) + require.NotEmpty(t, md) + assert.Contains(t, md, "apache.access.ssl.protocol") + assert.Contains(t, md, "SSL protocol version") + assert.Empty(t, hoverFieldReference(line, protocol.Position{Line: 0, Character: 2}, packageRoot, filePath)) + assert.Equal(t, "apache.access.ssl.protocol", extractFieldValueAtCursor(line, protocol.Position{Line: 0, Character: 12})) + assert.Empty(t, extractFieldValueAtCursor(line, protocol.Position{Line: 0, Character: 2})) +} + +func TestHoverManifestKeyAndFieldDefinitions(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + manifestPath := filepath.Join(packageRoot, "manifest.yml") + manifestText := readTestFile(t, manifestPath) + line := getLineAtText(manifestText, 2) + + md := hoverManifestKey(line, protocol.Position{Line: 2, Character: 1}, manifestPath, packageRoot, manifestText) + require.NotEmpty(t, md) + assert.Contains(t, md, "**title**") + assert.Empty(t, hoverManifestKey(line, protocol.Position{Line: 2, Character: 7}, manifestPath, packageRoot, manifestText)) + + fieldsPath := filepath.Join(packageRoot, "data_stream", "status", "fields", "fields.yml") + assert.Contains(t, hoverFieldDefinition("type: keyword", protocol.Position{Line: 0, Character: 8}, fieldsPath), "Exact-value string") + assert.Contains(t, hoverFieldDefinition("unit: byte", protocol.Position{Line: 0, Character: 7}, fieldsPath), "Data size in bytes") + assert.Contains(t, hoverFieldDefinition("metric_type: counter", protocol.Position{Line: 0, Character: 15}, fieldsPath), "cumulative metric") + assert.Empty(t, hoverFieldDefinition("type: keyword", protocol.Position{Line: 0, Character: 2}, fieldsPath)) + assert.Equal(t, "type", extractYAMLKey(" type: text")) +} + +func TestTextDocumentHoverUsesOpenBufferContent(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + server := NewServer() + + manifestPath := filepath.Join(packageRoot, "manifest.yml") + manifestText := readTestFile(t, manifestPath) + manifestURI := protocol.DocumentUri(pathToURI(manifestPath)) + server.documents.Set(manifestURI, manifestText) + + hover, err := server.textDocumentHover(nil, &protocol.HoverParams{ + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ + TextDocument: protocol.TextDocumentIdentifier{URI: manifestURI}, + Position: protocol.Position{Line: 2, Character: 1}, + }, + }) + require.NoError(t, err) + require.NotNil(t, hover) + + content, ok := hover.Contents.(protocol.MarkupContent) + require.True(t, ok) + assert.Contains(t, content.Value, "**title**") +} diff --git a/internal/lsp/live_diagnostics_test.go b/internal/lsp/live_diagnostics_test.go new file mode 100644 index 0000000000..a7aa9ca30e --- /dev/null +++ b/internal/lsp/live_diagnostics_test.go @@ -0,0 +1,204 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "io" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + protocol "github.com/tliron/glsp/protocol_3_16" +) + +func TestValidatePackageFSUsesUnsavedBufferText(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + manifestPath := filepath.Join(packageRoot, "manifest.yml") + + manifestText := readTestFile(t, manifestPath) + updatedText := strings.Replace(manifestText, "categories:\n - web", "# unsaved change\ncategories:\n - madeup", 1) + + diagsByFile := validatePackageFS(packageRoot, newOverlayFS(packageRoot, map[string]string{ + "manifest.yml": updatedText, + })) + + categoryDiag := findDiagnostic(diagsByFile[manifestPath], "field categories.0:") + require.NotNil(t, categoryDiag) + assert.Equal(t, uint32(11), categoryDiag.Range.Start.Line) +} + +func TestTextDocumentDidChangePublishesAndDidCloseClearsUnsavedDiagnostics(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + manifestPath := filepath.Join(packageRoot, "manifest.yml") + manifestURI := protocol.DocumentUri(pathToURI(manifestPath)) + + manifestText := readTestFile(t, manifestPath) + updatedText := strings.Replace(manifestText, "categories:\n - web", "# unsaved change\ncategories:\n - madeup", 1) + + server := NewServer() + t.Cleanup(server.debouncer.Shutdown) + + var mu sync.Mutex + var published []protocol.PublishDiagnosticsParams + + server.notifyMu.Lock() + server.notify = func(method string, params any) { + if method != protocol.ServerTextDocumentPublishDiagnostics { + return + } + + publish, ok := params.(protocol.PublishDiagnosticsParams) + if !ok { + return + } + + mu.Lock() + defer mu.Unlock() + published = append(published, publish) + } + server.notifyMu.Unlock() + + err := server.textDocumentDidChange(nil, &protocol.DidChangeTextDocumentParams{ + TextDocument: protocol.VersionedTextDocumentIdentifier{ + TextDocumentIdentifier: protocol.TextDocumentIdentifier{URI: manifestURI}, + Version: 1, + }, + ContentChanges: []any{ + protocol.TextDocumentContentChangeEventWhole{Text: updatedText}, + }, + }) + require.NoError(t, err) + + require.Eventually(t, func() bool { + publish, ok := latestDiagnosticsForURI(&mu, &published, manifestURI) + if !ok { + return false + } + + categoryDiag := findDiagnostic(publish.Diagnostics, "field categories.0:") + return categoryDiag != nil && categoryDiag.Range.Start.Line == 11 + }, 3*time.Second, 50*time.Millisecond) + + err = server.textDocumentDidClose(nil, &protocol.DidCloseTextDocumentParams{ + TextDocument: protocol.TextDocumentIdentifier{URI: manifestURI}, + }) + require.NoError(t, err) + + require.Eventually(t, func() bool { + publish, ok := latestDiagnosticsForURI(&mu, &published, manifestURI) + return ok && findDiagnostic(publish.Diagnostics, "field categories.0:") == nil + }, 3*time.Second, 50*time.Millisecond) +} + +func latestDiagnosticsForURI(mu *sync.Mutex, published *[]protocol.PublishDiagnosticsParams, uri protocol.DocumentUri) (protocol.PublishDiagnosticsParams, bool) { + mu.Lock() + defer mu.Unlock() + + for i := len(*published) - 1; i >= 0; i-- { + if (*published)[i].URI == uri { + return (*published)[i], true + } + } + + return protocol.PublishDiagnosticsParams{}, false +} + +func findDiagnostic(diags []protocol.Diagnostic, substring string) *protocol.Diagnostic { + for i := range diags { + if strings.Contains(diags[i].Message, substring) { + return &diags[i] + } + } + return nil +} + +func fixturePackagePath(t *testing.T, elems ...string) string { + t.Helper() + + _, thisFile, _, ok := runtime.Caller(0) + require.True(t, ok) + + repoRoot := filepath.Join(filepath.Dir(thisFile), "..", "..") + parts := append([]string{repoRoot}, elems...) + return filepath.Join(parts...) +} + +func copyFixturePackage(t *testing.T, src string) string { + t.Helper() + + dst := filepath.Join(t.TempDir(), filepath.Base(src)) + require.NoError(t, copyDir(src, dst)) + return dst +} + +func copyDir(src, dst string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + + if err := os.MkdirAll(dst, info.Mode()); err != nil { + return err + } + + entries, err := os.ReadDir(src) + if err != nil { + return err + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + if err := copyDir(srcPath, dstPath); err != nil { + return err + } + continue + } + + if err := copyFile(srcPath, dstPath); err != nil { + return err + } + } + + return nil +} + +func copyFile(src, dst string) error { + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + info, err := source.Stat() + if err != nil { + return err + } + + target, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) + if err != nil { + return err + } + defer target.Close() + + _, err = io.Copy(target, source) + return err +} + +func readTestFile(t *testing.T, path string) string { + t.Helper() + + data, err := os.ReadFile(path) + require.NoError(t, err) + return string(data) +} diff --git a/internal/lsp/overlay_fs_test.go b/internal/lsp/overlay_fs_test.go new file mode 100644 index 0000000000..b67fd99792 --- /dev/null +++ b/internal/lsp/overlay_fs_test.go @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "io" + "io/fs" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOverlayFSExposesVirtualFileMetadata(t *testing.T) { + fsys := newOverlayFS(t.TempDir(), map[string]string{ + "manifest.yml": "title: buffer", + }) + + file, err := fsys.Open("manifest.yml") + require.NoError(t, err) + defer file.Close() + + data, err := io.ReadAll(file) + require.NoError(t, err) + assert.Equal(t, "title: buffer", string(data)) + + info, err := file.Stat() + require.NoError(t, err) + assert.Equal(t, "manifest.yml", info.Name()) + assert.Equal(t, int64(len("title: buffer")), info.Size()) + assert.Equal(t, fs.FileMode(0o444), info.Mode()) + assert.Equal(t, time.Time{}, info.ModTime()) + assert.False(t, info.IsDir()) + assert.Nil(t, info.Sys()) +} diff --git a/internal/lsp/position_test.go b/internal/lsp/position_test.go index 940d906153..5414452dfc 100644 --- a/internal/lsp/position_test.go +++ b/internal/lsp/position_test.go @@ -52,12 +52,119 @@ version: 0.0.1 assert.Equal(t, uint32(2), r.Start.Line, "expected line 2 (0-based) for title") } +func TestFindPosition_RootAdditionalProperty(t *testing.T) { + f := writePositionTestFile(t, `format_version: 3.3.0 +license: basic +`) + + r := findPosition(f, "field (root): Additional property license is not allowed") + assert.Equal(t, uint32(1), r.Start.Line, "expected line 1 (0-based) for license") +} + +func TestFindPosition_DottedAdditionalProperty(t *testing.T) { + f := writePositionTestFile(t, `conditions: + kibana.version: "^8.0.0" + elastic.subscription: basic +`) + + r := findPosition(f, "field conditions: Additional property kibana.version is not allowed") + assert.Equal(t, uint32(1), r.Start.Line, "expected line 1 (0-based) for kibana.version") + + r = findPosition(f, "field conditions: Additional property elastic.subscription is not allowed") + assert.Equal(t, uint32(2), r.Start.Line, "expected line 2 (0-based) for elastic.subscription") +} + +func TestFindPosition_HyphenatedSegment(t *testing.T) { + f := writePositionTestFile(t, `services: + docker-custom-agent: + image: foo +`) + + r := findPosition(f, "field services.docker-custom-agent: Must not be present") + assert.Equal(t, uint32(1), r.Start.Line, "expected line 1 (0-based) for docker-custom-agent") +} + +func TestFindPosition_LineNumberMessage(t *testing.T) { + f := writePositionTestFile(t, `processors: + - set: + field: key1 + value: value1 +`) + + r := findPosition(f, `set processor at line 2 missing required tag (SVR00006)`) + assert.Equal(t, uint32(1), r.Start.Line, "expected line 1 (0-based) for line-based diagnostic") +} + +func TestFindPosition_JSONDanglingReference(t *testing.T) { + f := writeJSONPositionTestFile(t, `{ + "references": [ + { + "id": "missing-ref", + "type": "search" + } + ] +} +`) + + r := findPosition(f, `dangling reference found: missing-ref (search)`) + assert.Equal(t, uint32(3), r.Start.Line, "expected line 3 (0-based) for reference id") +} + +func TestFindPosition_JSONDashboardReference(t *testing.T) { + f := writeJSONPositionTestFile(t, `{ + "references": [ + { + "id": "by-reference-vis", + "type": "visualization" + } + ] +} +`) + + r := findPosition(f, `reference found in dashboard: by-reference-vis (visualization)`) + assert.Equal(t, uint32(3), r.Start.Line, "expected line 3 (0-based) for dashboard reference id") +} + +func TestFindPosition_JSONLegacyVisualization(t *testing.T) { + f := writeJSONPositionTestFile(t, `{ + "attributes": { + "panelsJSON": [ + { + "title": "TSVB time series", + "type": "visualization" + } + ], + "title": "Dashboard with mixed by-value visualizations" + } +} +`) + + r := findPosition(f, `"Dashboard with mixed by-value visualizations" contains legacy visualization: "TSVB time series" (timeseries, TSVB)`) + assert.Equal(t, uint32(4), r.Start.Line, "expected line 4 (0-based) for visualization title") +} + func TestFindPosition_NoMatch(t *testing.T) { - dir := t.TempDir() - f := filepath.Join(dir, "manifest.yml") - require.NoError(t, os.WriteFile(f, []byte(`name: test -`), 0644)) + f := writePositionTestFile(t, `name: test +`) r := findPosition(f, "some error without field path") assert.Equal(t, uint32(0), r.Start.Line) } + +func writePositionTestFile(t *testing.T, contents string) string { + t.Helper() + + dir := t.TempDir() + f := filepath.Join(dir, "manifest.yml") + require.NoError(t, os.WriteFile(f, []byte(contents), 0644)) + return f +} + +func writeJSONPositionTestFile(t *testing.T, contents string) string { + t.Helper() + + dir := t.TempDir() + f := filepath.Join(dir, "asset.json") + require.NoError(t, os.WriteFile(f, []byte(contents), 0644)) + return f +} diff --git a/internal/lsp/server_test.go b/internal/lsp/server_test.go new file mode 100644 index 0000000000..5f2416b0c7 --- /dev/null +++ b/internal/lsp/server_test.go @@ -0,0 +1,64 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tliron/glsp" + protocol "github.com/tliron/glsp/protocol_3_16" +) + +func TestServerInitializeAndLifecycle(t *testing.T) { + server := NewServer() + + result, err := server.initialize(&glsp.Context{ + Notify: func(method string, params any) {}, + }, &protocol.InitializeParams{}) + require.NoError(t, err) + + initializeResult, ok := result.(protocol.InitializeResult) + require.True(t, ok) + require.NotNil(t, initializeResult.Capabilities.TextDocumentSync) + + assert.True(t, initializeResult.Capabilities.HoverProvider.(bool)) + assert.NotNil(t, initializeResult.Capabilities.CompletionProvider) + assert.NotNil(t, server.notify) + assert.NoError(t, server.initialized(nil, &protocol.InitializedParams{})) + assert.NoError(t, server.setTrace(nil, &protocol.SetTraceParams{Value: protocol.TraceValueMessage})) + assert.NoError(t, server.shutdown(nil)) +} + +func TestServerOpenAndSaveHandlers(t *testing.T) { + packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) + manifestPath := filepath.Join(packageRoot, "manifest.yml") + manifestURI := protocol.DocumentUri(pathToURI(manifestPath)) + + server := NewServer() + server.notifyMu.Lock() + server.notify = func(method string, params any) {} + server.notifyMu.Unlock() + t.Cleanup(server.debouncer.Shutdown) + + err := server.textDocumentDidOpen(nil, &protocol.DidOpenTextDocumentParams{ + TextDocument: protocol.TextDocumentItem{ + URI: manifestURI, + Text: "title", + }, + }) + require.NoError(t, err) + + text, ok := server.documents.Text(manifestPath) + require.True(t, ok) + assert.Equal(t, "title", text) + + err = server.textDocumentDidSave(nil, &protocol.DidSaveTextDocumentParams{ + TextDocument: protocol.TextDocumentIdentifier{URI: manifestURI}, + }) + require.NoError(t, err) +} diff --git a/internal/lsp/workspace_test.go b/internal/lsp/workspace_test.go new file mode 100644 index 0000000000..8488324b8c --- /dev/null +++ b/internal/lsp/workspace_test.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lsp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestURIPathRoundTripEncodesSpaces(t *testing.T) { + path := "/tmp/elastic package/manifest.yml" + uri := pathToURI(path) + + assert.Equal(t, "file:///tmp/elastic%20package/manifest.yml", uri) + + roundTripped, err := uriToPath(uri) + require.NoError(t, err) + assert.Equal(t, path, roundTripped) +} + +func TestURIToPathRejectsNonFileScheme(t *testing.T) { + _, err := uriToPath("https://example.com/manifest.yml") + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported URI scheme") +} From 55a241bbd924095238110fc70f4dd9751989f9dc Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Tue, 24 Mar 2026 13:51:01 +0100 Subject: [PATCH 07/12] =?UTF-8?q?=F0=9F=92=9A=20fix:=20resolve=20staticche?= =?UTF-8?q?ck=20lint=20and=20goimports=20formatting=20issues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove unused validatePackage function, range over string directly instead of []rune(string), align struct tags, regenerate README. Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 80 +----------------------------------- internal/lsp/diagnostics.go | 8 ---- internal/lsp/document.go | 2 +- internal/lsp/fields_index.go | 12 +++--- 4 files changed, 8 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index e5bb4e2aab..9c3c09a0d7 100644 --- a/README.md +++ b/README.md @@ -446,85 +446,7 @@ _Context: global_ Start a Language Server Protocol (LSP) server for Elastic integration packages. The LSP server communicates over stdin/stdout and provides real-time validation -diagnostics for integration packages opened in supported editors. When you open -or save any file inside a package, the server runs the same validation as -`elastic-package lint` and pushes diagnostics (errors) back to your editor. - -Flags: -- `--log-file ` — write debug logs to a file (useful for troubleshooting). - -#### Demo - -Run the included demo script to see the LSP in action without an editor: - -``` -$ ./scripts/lsp-demo.sh ./elastic-package - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Scenario 1: Valid package (apache) -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - - File: manifest.yml - (no errors) - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Scenario 2: Broken package (invalid name, missing files) -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - - File: data_stream/mystream/manifest.yml - [ERROR] expecting to find [fields] folder in ... - - File: manifest.yml - [ERROR] field owner: type is required - [ERROR] field name: Does not match pattern '^[a-z0-9_]+$' - [ERROR] expecting to find [changelog.yml] file in ... - [ERROR] expecting to find [docs] folder in ... - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Scenario 3: Open data_stream file — LSP discovers package root -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - - (same errors as Scenario 2 — the LSP walks up to find the package root) -``` - -The script creates a temporary broken package, sends JSON-RPC messages to the -LSP, and pretty-prints the diagnostics it receives back. - -#### Zed editor extension - -A companion Zed extension is available at -[`zed-elastic-package`](https://github.com/elastic/zed-elastic-package). To -install it for development: - -1. Build elastic-package with the `lsp` command and ensure it is on your `PATH`: - ``` - go build -o elastic-package . - ``` - -2. Clone the extension repo: - ``` - git clone https://github.com/elastic/zed-elastic-package.git - ``` - -3. In Zed, open the command palette and run **zed: Install Dev Extension**, - then select the `zed-elastic-package` directory. - -4. Open any integration package. YAML and JSON files will automatically get - validation diagnostics from the LSP. - -To point the extension at a custom binary path, add this to your Zed settings: - -```json -{ - "lsp": { - "elastic-package-lsp": { - "binary": { - "path": "/path/to/elastic-package" - } - } - } -} -``` +diagnostics for integration packages opened in supported editors. ### `elastic-package modify` diff --git a/internal/lsp/diagnostics.go b/internal/lsp/diagnostics.go index 0b49ec66be..58f3947a6e 100644 --- a/internal/lsp/diagnostics.go +++ b/internal/lsp/diagnostics.go @@ -39,14 +39,6 @@ var ( errorCodeRe = regexp.MustCompile(`\(([A-Z]{2,3}\d{5})\)$`) ) -// validatePackage runs validation on the package at the given root path and -// returns diagnostics grouped by absolute file path. -func validatePackage(packageRoot string) map[string][]protocol.Diagnostic { - return validatePackageWith(packageRoot, os.DirFS(packageRoot), func() (error, error) { - return validation.ValidateAndFilterFromPath(packageRoot) - }) -} - func validatePackageFS(packageRoot string, fsys fs.FS) map[string][]protocol.Diagnostic { return validatePackageWith(packageRoot, fsys, func() (error, error) { return validation.ValidateAndFilterFromFS(packageRoot, fsys) diff --git a/internal/lsp/document.go b/internal/lsp/document.go index d2fc92a86c..024c47054e 100644 --- a/internal/lsp/document.go +++ b/internal/lsp/document.go @@ -159,7 +159,7 @@ func utf16ColumnToRuneOffset(line string, target int) int { offset := 0 column := 0 - for _, r := range []rune(line) { + for _, r := range line { if column >= target { return offset } diff --git a/internal/lsp/fields_index.go b/internal/lsp/fields_index.go index 1e497b10ae..a02212c330 100644 --- a/internal/lsp/fields_index.go +++ b/internal/lsp/fields_index.go @@ -14,12 +14,12 @@ import ( // FieldInfo holds metadata about a single field from fields/*.yml. type FieldInfo struct { - Name string `yaml:"name"` - Type string `yaml:"type"` - Description string `yaml:"description"` - Unit string `yaml:"unit,omitempty"` - MetricType string `yaml:"metric_type,omitempty"` - External string `yaml:"external,omitempty"` + Name string `yaml:"name"` + Type string `yaml:"type"` + Description string `yaml:"description"` + Unit string `yaml:"unit,omitempty"` + MetricType string `yaml:"metric_type,omitempty"` + External string `yaml:"external,omitempty"` Fields []FieldInfo `yaml:"fields,omitempty"` } From b1b3da82fd87ff54fed285e49182a6961f46a7da Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Wed, 25 Mar 2026 14:01:25 +0100 Subject: [PATCH 08/12] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20schema-driven?= =?UTF-8?q?=20nested=20completions=20and=20value=20suggestions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Manifest completions now resolve nested YAML context (e.g. completing keys inside policy_templates.inputs or streams.vars). Value completions suggest enum values and booleans from the package-spec JSON schemas. Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/lsp/completion.go | 220 +++++++++++++++++++- internal/lsp/completion_test.go | 188 ++++++++++++++++- internal/lsp/manifest_schema.go | 289 +++++++++++++++++++++++++++ internal/lsp/manifest_schema_test.go | 18 ++ 4 files changed, 699 insertions(+), 16 deletions(-) diff --git a/internal/lsp/completion.go b/internal/lsp/completion.go index 03f041791a..47d17b46db 100644 --- a/internal/lsp/completion.go +++ b/internal/lsp/completion.go @@ -14,6 +14,22 @@ import ( protocol "github.com/tliron/glsp/protocol_3_16" ) +type manifestCompletionMode int + +const ( + manifestCompletionModeKey manifestCompletionMode = iota + manifestCompletionModeValue +) + +type manifestCompletionContext struct { + mode manifestCompletionMode + path string + prefix string + currentIndent int + parentIndent int + listItemPrefix bool +} + func (s *Server) textDocumentCompletion(ctx *glsp.Context, params *protocol.CompletionParams) (any, error) { filePath, err := uriToPath(params.TextDocument.URI) if err != nil { @@ -35,7 +51,7 @@ func (s *Server) textDocumentCompletion(ctx *glsp.Context, params *protocol.Comp // Completing a "field: " value in a pipeline — suggest field names. items = s.completeFieldNames(packageRoot, filePath, line) case isManifestFile(filePath, packageRoot): - items = completeManifestKeys(filePath, packageRoot, line, documentText) + items = completeManifestItems(filePath, packageRoot, documentText, params.Position) case isFieldsDefinitionFile(filePath): items = completeFieldTypeValues(line) } @@ -129,27 +145,74 @@ func completeFieldTypeValues(line string) []protocol.CompletionItem { return items } -// completeManifestKeys suggests top-level manifest keys based on package-spec. -func completeManifestKeys(filePath, packageRoot, line, documentText string) []protocol.CompletionItem { - trimmed := strings.TrimSpace(line) - // Only suggest at start of a line (not inside a value). - if strings.Contains(trimmed, ":") { +// completeManifestItems suggests manifest keys or values based on package-spec. +func completeManifestItems(filePath, packageRoot, documentText string, pos protocol.Position) []protocol.CompletionItem { + kind := manifestSchemaKindForFile(filePath, packageRoot, documentText) + context, ok := resolveManifestCompletionContext(documentText, pos) + if !ok { return nil } - keys := manifestTopLevelKeys(manifestSchemaKindForFile(filePath, packageRoot, documentText)) + switch context.mode { + case manifestCompletionModeKey: + return completeManifestKeyItems(kind, context) + case manifestCompletionModeValue: + return completeManifestValueItems(kind, context) + default: + return nil + } +} + +func completeManifestKeyItems(kind manifestSchemaKind, context manifestCompletionContext) []protocol.CompletionItem { + keys, fromArray := manifestChildKeys(context.path, kind) + if len(keys) == 0 { + return nil + } + + insertPrefix := "" + if fromArray && !context.listItemPrefix && context.parentIndent >= 0 && + context.currentIndent == context.parentIndent+2 { + insertPrefix = "- " + } propKind := protocol.CompletionItemKindProperty var items []protocol.CompletionItem for _, k := range keys { - if trimmed != "" && !strings.HasPrefix(k, trimmed) { + if context.prefix != "" && !strings.HasPrefix(k, context.prefix) { continue } label := k - items = append(items, protocol.CompletionItem{ + item := protocol.CompletionItem{ Label: label, Kind: &propKind, - InsertText: strPtr(k + ": "), + InsertText: strPtr(insertPrefix + k + ": "), + } + if doc := manifestDoc(joinManifestPath(context.path, k), kind); doc != "" { + item.Documentation = &protocol.MarkupContent{ + Kind: protocol.MarkupKindMarkdown, + Value: doc, + } + } + items = append(items, item) + } + return items +} + +func completeManifestValueItems(kind manifestSchemaKind, context manifestCompletionContext) []protocol.CompletionItem { + values := manifestValueCandidates(context.path, kind) + if len(values) == 0 { + return nil + } + + enumKind := protocol.CompletionItemKindEnumMember + var items []protocol.CompletionItem + for _, value := range values { + if context.prefix != "" && !strings.HasPrefix(value, context.prefix) { + continue + } + items = append(items, protocol.CompletionItem{ + Label: value, + Kind: &enumKind, }) } return items @@ -207,6 +270,129 @@ func dataStreamFromPath(filePath, packageRoot string) string { return "" } +func resolveManifestCompletionContext(documentText string, pos protocol.Position) (manifestCompletionContext, bool) { + lines := splitLines(documentText) + lineNum := int(pos.Line) + if lineNum < 0 || lineNum >= len(lines) { + return manifestCompletionContext{}, false + } + + linePrefix := linePrefixAtPosition(lines[lineNum], pos) + trimmed := strings.TrimSpace(linePrefix) + if strings.HasPrefix(trimmed, "#") { + return manifestCompletionContext{}, false + } + + if context, ok := resolveManifestValueContext(lines, lineNum, linePrefix); ok { + return context, true + } + + return resolveManifestKeyContext(lines, lineNum, linePrefix), true +} + +func resolveManifestValueContext(lines []string, lineNum int, linePrefix string) (manifestCompletionContext, bool) { + key, _, _ := yamlKeyDetails(linePrefix) + if key == "" || !strings.Contains(linePrefix, ":") { + return manifestCompletionContext{}, false + } + + path, parentIndent := resolveManifestParentPath(lines, lineNum, yamlIndent(linePrefix)) + prefix := "" + if value, _, _, ok := valueAfterKey(linePrefix, key+":"); ok { + prefix = value + } + + return manifestCompletionContext{ + mode: manifestCompletionModeValue, + path: joinManifestPath(strings.Join(path, "."), key), + prefix: prefix, + currentIndent: yamlIndent(linePrefix), + parentIndent: parentIndent, + listItemPrefix: hasListItemPrefix(linePrefix), + }, true +} + +func resolveManifestKeyContext(lines []string, lineNum int, linePrefix string) manifestCompletionContext { + path, parentIndent := resolveManifestParentPath(lines, lineNum, yamlIndent(linePrefix)) + + return manifestCompletionContext{ + mode: manifestCompletionModeKey, + path: strings.Join(path, "."), + prefix: extractManifestKeyPrefix(linePrefix), + currentIndent: yamlIndent(linePrefix), + parentIndent: parentIndent, + listItemPrefix: hasListItemPrefix(linePrefix), + } +} + +func resolveManifestParentPath(lines []string, lineNum, currentIndent int) ([]string, int) { + currentIndent = max(currentIndent, 0) + + var path []string + parentIndent := -1 + indentLimit := currentIndent + for i := lineNum - 1; i >= 0; i-- { + line := lines[i] + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + indent := yamlIndent(line) + if indent >= indentLimit { + continue + } + + key, isListItem, hasInlineValue := yamlKeyDetails(line) + if key == "" { + continue + } + if isListItem && hasInlineValue { + continue + } + + if parentIndent < 0 { + parentIndent = indent + } + path = append([]string{key}, path...) + indentLimit = indent + if indent == 0 { + break + } + } + + return path, parentIndent +} + +func linePrefixAtPosition(line string, pos protocol.Position) string { + runes := []rune(line) + offset := utf16ColumnToRuneOffset(line, int(pos.Character)) + offset = min(offset, len(runes)) + return string(runes[:offset]) +} + +func extractManifestKeyPrefix(linePrefix string) string { + trimmed := strings.TrimSpace(linePrefix) + if strings.HasPrefix(trimmed, "-") { + trimmed = strings.TrimSpace(strings.TrimPrefix(trimmed, "-")) + } + if colonIdx := strings.Index(trimmed, ":"); colonIdx >= 0 { + trimmed = trimmed[:colonIdx] + } + return strings.TrimSpace(trimmed) +} + +func joinManifestPath(base, child string) string { + if base == "" { + return child + } + return base + "." + child +} + +func hasListItemPrefix(line string) bool { + return strings.HasPrefix(strings.TrimSpace(line), "-") +} + // formatFieldDetail returns a short detail string for a field. func formatFieldDetail(f FieldInfo) string { parts := []string{f.Type} @@ -218,3 +404,17 @@ func formatFieldDetail(f FieldInfo) string { } return strings.Join(parts, ", ") } + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/internal/lsp/completion_test.go b/internal/lsp/completion_test.go index 106f2be32b..c58e174ac9 100644 --- a/internal/lsp/completion_test.go +++ b/internal/lsp/completion_test.go @@ -6,6 +6,7 @@ package lsp import ( "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -33,9 +34,45 @@ func TestCompletionHelpers(t *testing.T) { Unit: "byte", MetricType: "counter", })) + assert.Equal(t, "ty", extractManifestKeyPrefix(" - ty")) } -func TestCompleteFieldNamesAndManifestKeys(t *testing.T) { +func TestResolveManifestCompletionContext(t *testing.T) { + documentText, pos := completionDocument(t, `policy_templates: + | +`) + context, ok := resolveManifestCompletionContext(documentText, pos) + require.True(t, ok) + assert.Equal(t, manifestCompletionModeKey, context.mode) + assert.Equal(t, "policy_templates", context.path) + assert.Equal(t, "", context.prefix) + assert.Equal(t, 2, context.currentIndent) + assert.Equal(t, 0, context.parentIndent) + assert.False(t, context.listItemPrefix) + + documentText, pos = completionDocument(t, `policy_templates: + - name: demo + inputs: + - ty| +`) + context, ok = resolveManifestCompletionContext(documentText, pos) + require.True(t, ok) + assert.Equal(t, manifestCompletionModeKey, context.mode) + assert.Equal(t, "policy_templates.inputs", context.path) + assert.Equal(t, "ty", context.prefix) + assert.True(t, context.listItemPrefix) + + documentText, pos = completionDocument(t, `owner: + type: el| +`) + context, ok = resolveManifestCompletionContext(documentText, pos) + require.True(t, ok) + assert.Equal(t, manifestCompletionModeValue, context.mode) + assert.Equal(t, "owner.type", context.path) + assert.Equal(t, "el", context.prefix) +} + +func TestCompleteFieldNamesAndManifestItems(t *testing.T) { packageRoot := copyFixturePackage(t, fixturePackagePath(t, "test", "packages", "parallel", "apache")) server := NewServer() @@ -54,7 +91,8 @@ func TestCompleteFieldNamesAndManifestKeys(t *testing.T) { assert.Equal(t, "long, unit: byte, metric: counter", *statusItem.Detail) manifestPath := filepath.Join(packageRoot, "manifest.yml") - manifestItems := completeManifestKeys(manifestPath, packageRoot, "pol", readTestFile(t, manifestPath)) + documentText, pos := completionDocument(t, "pol|") + manifestItems := completeManifestItems(manifestPath, packageRoot, documentText, pos) require.NotEmpty(t, manifestItems) item := findCompletionItem(manifestItems, "policy_templates") require.NotNil(t, item) @@ -62,7 +100,119 @@ func TestCompleteFieldNamesAndManifestKeys(t *testing.T) { assert.Equal(t, "policy_templates: ", *item.InsertText) assert.Nil(t, findCompletionItem(manifestItems, "title")) - assert.Nil(t, completeManifestKeys(manifestPath, packageRoot, "title: Apache", readTestFile(t, manifestPath))) + documentText, pos = completionDocument(t, "title: Apache|") + assert.Nil(t, completeManifestItems(manifestPath, packageRoot, documentText, pos)) +} + +func TestCompleteManifestItemsSupportsNestedKeys(t *testing.T) { + packageRoot := "/tmp/pkg" + manifestPath := filepath.Join(packageRoot, "manifest.yml") + dataStreamManifestPath := filepath.Join(packageRoot, "data_stream", "logs", "manifest.yml") + + documentText, pos := completionDocument(t, `type: input +policy_templates: + | +`) + items := completeManifestItems(manifestPath, packageRoot, documentText, pos) + item := findCompletionItem(items, "name") + require.NotNil(t, item) + require.NotNil(t, item.InsertText) + assert.Equal(t, "- name: ", *item.InsertText) + + documentText, pos = completionDocument(t, `type: integration +policy_templates: + - name: demo + title: Demo + description: Demo + inputs: + - ty| +`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + item = findCompletionItem(items, "type") + require.NotNil(t, item) + require.NotNil(t, item.InsertText) + assert.Equal(t, "type: ", *item.InsertText) + + documentText, pos = completionDocument(t, `type: integration +policy_templates: + - name: demo + title: Demo + description: Demo + inputs: + - | +`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "vars")) + + documentText, pos = completionDocument(t, `type: input +vars: + - req| +`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "required")) + + documentText, pos = completionDocument(t, `type: input +policy_templates: + - name: demo + title: Demo + description: Demo + input: otelcol + vars: + - sho| +`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "show_user")) + + documentText, pos = completionDocument(t, `title: Demo +streams: + - en| +`) + items = completeManifestItems(dataStreamManifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "enabled")) +} + +func TestCompleteManifestItemsSuggestsSchemaValues(t *testing.T) { + packageRoot := "/tmp/pkg" + manifestPath := filepath.Join(packageRoot, "manifest.yml") + dataStreamManifestPath := filepath.Join(packageRoot, "data_stream", "logs", "manifest.yml") + + documentText, pos := completionDocument(t, `owner: + type: el| +`) + items := completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "elastic")) + + documentText, pos = completionDocument(t, `type: integration +policy_templates: + - name: demo + - name: second +policy_templates_behavior: c| +`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "combined_policy")) + assert.Nil(t, findCompletionItem(items, "all")) + + documentText, pos = completionDocument(t, `type: input +vars: + - type: bo| +`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "bool")) + + documentText, pos = completionDocument(t, `type: input +vars: + - required: f| +`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "false")) + assert.Nil(t, findCompletionItem(items, "true")) + + documentText, pos = completionDocument(t, `title: Demo +streams: + - enabled: t| +`) + items = completeManifestItems(dataStreamManifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "true")) } func TestCompleteFieldTypeValuesSuggestsKnownTypes(t *testing.T) { @@ -81,18 +231,18 @@ func TestTextDocumentCompletionUsesOpenBufferContent(t *testing.T) { manifestPath := filepath.Join(packageRoot, "manifest.yml") manifestURI := protocol.DocumentUri(pathToURI(manifestPath)) - server.documents.Set(manifestURI, "title") + server.documents.Set(manifestURI, "type: input\nvars:\n - req") result, err := server.textDocumentCompletion(nil, &protocol.CompletionParams{ TextDocumentPositionParams: protocol.TextDocumentPositionParams{ TextDocument: protocol.TextDocumentIdentifier{URI: manifestURI}, - Position: protocol.Position{Line: 0, Character: 3}, + Position: protocol.Position{Line: 2, Character: 7}, }, }) require.NoError(t, err) manifestItems, ok := result.([]protocol.CompletionItem) require.True(t, ok) - assert.NotNil(t, findCompletionItem(manifestItems, "title")) + assert.NotNil(t, findCompletionItem(manifestItems, "required")) fieldsPath := filepath.Join(packageRoot, "data_stream", "access", "fields", "tmp.yml") fieldsURI := protocol.DocumentUri(pathToURI(fieldsPath)) @@ -118,3 +268,29 @@ func findCompletionItem(items []protocol.CompletionItem, label string) *protocol } return nil } + +func completionDocument(t *testing.T, marked string) (string, protocol.Position) { + t.Helper() + + lines := strings.Split(marked, "\n") + for lineNum, line := range lines { + if idx := strings.IndexRune(line, '|'); idx >= 0 { + lines[lineNum] = strings.Replace(line, "|", "", 1) + return strings.Join(lines, "\n"), protocol.Position{ + Line: uint32(lineNum), + Character: uint32(utf16Column(line[:idx])), + } + } + } + + t.Fatal("missing cursor marker") + return "", protocol.Position{} +} + +func utf16Column(s string) int { + column := 0 + for _, r := range s { + column += utf16Width(r) + } + return column +} diff --git a/internal/lsp/manifest_schema.go b/internal/lsp/manifest_schema.go index 6b2765afb4..ef73c0bf23 100644 --- a/internal/lsp/manifest_schema.go +++ b/internal/lsp/manifest_schema.go @@ -68,6 +68,22 @@ func manifestTopLevelKeys(kind manifestSchemaKind) []string { return keys } +func manifestChildKeys(dottedPath string, kind manifestSchemaKind) ([]string, bool) { + keys, fromArray, err := schemaLoader.childPropertiesForPath(string(kind), dottedPath) + if err != nil { + return nil, false + } + return keys, fromArray +} + +func manifestValueCandidates(dottedPath string, kind manifestSchemaKind) []string { + values, err := schemaLoader.valueCandidatesForPath(string(kind), dottedPath) + if err != nil { + return nil + } + return values +} + func manifestDoc(dottedPath string, kind manifestSchemaKind) string { doc, err := schemaLoader.docForPath(string(kind), dottedPath) if err != nil { @@ -133,6 +149,61 @@ func (l *manifestSchemaLoader) docForPath(schemaPath, dottedPath string) (string return formatManifestDoc(dottedPath, node, required), nil } +func (l *manifestSchemaLoader) childPropertiesForPath(schemaPath, dottedPath string) ([]string, bool, error) { + nodes, err := l.nodesForPath(schemaPath, dottedPath) + if err != nil { + return nil, false, err + } + + keys := make(map[string]struct{}) + var fromArray bool + for _, node := range nodes { + childKeys, childFromArray, err := l.childProperties(node.path, node.node) + if err != nil { + continue + } + fromArray = fromArray || childFromArray + for _, key := range childKeys { + keys[key] = struct{}{} + } + } + + if len(keys) == 0 { + return nil, fromArray, nil + } + + out := make([]string, 0, len(keys)) + for key := range keys { + out = append(out, key) + } + sort.Strings(out) + return out, fromArray, nil +} + +func (l *manifestSchemaLoader) valueCandidatesForPath(schemaPath, dottedPath string) ([]string, error) { + nodes, err := l.nodesForPath(schemaPath, dottedPath) + if err != nil { + return nil, err + } + + var out []string + seen := make(map[string]struct{}) + for _, node := range nodes { + values, err := l.valueCandidates(node.path, node.node) + if err != nil { + continue + } + for _, value := range values { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + } + return out, nil +} + func (l *manifestSchemaLoader) child(schemaPath string, node any, segment string) (string, any, bool, error) { schemaPath, current, err := l.normalize(schemaPath, node) if err != nil { @@ -162,6 +233,200 @@ func (l *manifestSchemaLoader) child(schemaPath string, node any, segment string return "", nil, false, fmt.Errorf("schema path not found: %s", segment) } +type manifestSchemaNode struct { + path string + node map[string]any +} + +func (l *manifestSchemaLoader) nodesForPath(schemaPath, dottedPath string) ([]manifestSchemaNode, error) { + root, err := l.load(schemaPath) + if err != nil { + return nil, err + } + + nodes, err := l.expandNodes(schemaPath, root) + if err != nil { + return nil, err + } + + if dottedPath == "" { + return nodes, nil + } + + for _, segment := range strings.Split(dottedPath, ".") { + if segment == "" { + continue + } + + var next []manifestSchemaNode + for _, node := range nodes { + children, err := l.childNodes(node.path, node.node, segment) + if err != nil { + continue + } + next = append(next, children...) + } + + if len(next) == 0 { + return nil, fmt.Errorf("schema path not found: %s", dottedPath) + } + nodes = next + } + + return nodes, nil +} + +func (l *manifestSchemaLoader) expandNodes(schemaPath string, node any) ([]manifestSchemaNode, error) { + schemaPath, current, err := l.normalize(schemaPath, node) + if err != nil { + return nil, err + } + + nodes := []manifestSchemaNode{{path: schemaPath, node: current}} + for _, key := range []string{"allOf", "oneOf", "anyOf", "then", "else"} { + for _, branch := range schemaBranches(current[key]) { + branchNodes, err := l.expandNodes(schemaPath, branch) + if err != nil { + continue + } + nodes = append(nodes, branchNodes...) + } + } + + return nodes, nil +} + +func (l *manifestSchemaLoader) childNodes(schemaPath string, node map[string]any, segment string) ([]manifestSchemaNode, error) { + if props := asMap(node["properties"]); props != nil { + if child, ok := props[segment]; ok { + return l.expandNodes(schemaPath, child) + } + } + + if items, ok := node["items"]; ok { + itemNodes, err := l.expandNodes(schemaPath, items) + if err != nil { + return nil, err + } + + var children []manifestSchemaNode + for _, itemNode := range itemNodes { + itemChildren, err := l.childNodes(itemNode.path, itemNode.node, segment) + if err != nil { + continue + } + children = append(children, itemChildren...) + } + if len(children) > 0 { + return children, nil + } + } + + return nil, fmt.Errorf("schema path not found: %s", segment) +} + +func (l *manifestSchemaLoader) childProperties(schemaPath string, node map[string]any) ([]string, bool, error) { + schemaPath, current, err := l.normalize(schemaPath, node) + if err != nil { + return nil, false, err + } + + keys := make(map[string]struct{}) + for key := range asMap(current["properties"]) { + keys[key] = struct{}{} + } + + for _, branchKey := range []string{"allOf", "oneOf", "anyOf", "then", "else"} { + for _, branch := range schemaBranches(current[branchKey]) { + branchKeys, _, err := l.childProperties(schemaPath, asMap(branch)) + if err != nil { + continue + } + for _, key := range branchKeys { + keys[key] = struct{}{} + } + } + } + + if len(keys) > 0 { + out := make([]string, 0, len(keys)) + for key := range keys { + out = append(out, key) + } + sort.Strings(out) + return out, false, nil + } + + if items, ok := current["items"]; ok { + itemNodes, err := l.expandNodes(schemaPath, items) + if err != nil { + return nil, false, err + } + + for _, itemNode := range itemNodes { + itemKeys, _, err := l.childProperties(itemNode.path, itemNode.node) + if err != nil { + continue + } + for _, key := range itemKeys { + keys[key] = struct{}{} + } + } + } + + out := make([]string, 0, len(keys)) + for key := range keys { + out = append(out, key) + } + sort.Strings(out) + return out, len(out) > 0, nil +} + +func (l *manifestSchemaLoader) valueCandidates(schemaPath string, node map[string]any) ([]string, error) { + _, current, err := l.normalize(schemaPath, node) + if err != nil { + return nil, err + } + + var out []string + seen := make(map[string]struct{}) + for _, value := range schemaEnum(current) { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + + if schemaHasType(current, "boolean") { + for _, value := range []string{"true", "false"} { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + } + + for _, branchKey := range []string{"allOf", "oneOf", "anyOf", "then", "else"} { + for _, branch := range schemaBranches(current[branchKey]) { + branchValues, err := l.valueCandidates(schemaPath, asMap(branch)) + if err != nil { + continue + } + for _, value := range branchValues { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + } + } + + return out, nil +} + func (l *manifestSchemaLoader) normalize(schemaPath string, node any) (string, map[string]any, error) { current := cloneMap(asMap(node)) if current == nil { @@ -304,6 +569,20 @@ func schemaEnum(node map[string]any) []string { return out } +func schemaHasType(node map[string]any, target string) bool { + switch value := node["type"].(type) { + case string: + return value == target + case []any: + for _, item := range value { + if str, ok := item.(string); ok && str == target { + return true + } + } + } + return false +} + func navigate(node any, segment string) (any, bool) { switch current := node.(type) { case map[string]any: @@ -336,6 +615,16 @@ func asSlice(value any) []any { return nil } +func schemaBranches(value any) []any { + if list := asSlice(value); list != nil { + return list + } + if value == nil { + return nil + } + return []any{value} +} + func asStringSlice(value any) []string { list := asSlice(value) out := make([]string, 0, len(list)) diff --git a/internal/lsp/manifest_schema_test.go b/internal/lsp/manifest_schema_test.go index fbd345225d..3a21be7939 100644 --- a/internal/lsp/manifest_schema_test.go +++ b/internal/lsp/manifest_schema_test.go @@ -43,6 +43,24 @@ func TestManifestDocCoversNestedPolicyTemplateFields(t *testing.T) { assert.Contains(t, doc, "deployment mode") } +func TestManifestCompletionSchemaHelpers(t *testing.T) { + keys, fromArray := manifestChildKeys("policy_templates", manifestSchemaIntegration) + require.NotEmpty(t, keys) + assert.True(t, fromArray) + assert.Contains(t, keys, "inputs") + assert.Contains(t, keys, "name") + + keys, fromArray = manifestChildKeys("policy_templates.inputs", manifestSchemaIntegration) + require.NotEmpty(t, keys) + assert.True(t, fromArray) + assert.Contains(t, keys, "type") + assert.Contains(t, keys, "vars") + + assert.Contains(t, manifestValueCandidates("owner.type", manifestSchemaIntegration), "elastic") + assert.Contains(t, manifestValueCandidates("policy_templates_behavior", manifestSchemaIntegration), "combined_policy") + assert.Contains(t, manifestValueCandidates("vars.required", manifestSchemaInput), "true") +} + func TestResolveYAMLPathFromDocumentText(t *testing.T) { path := resolveYAMLPath(`policy_templates: - name: apache From 8563c4ab193631ce1a881d15de850a249cb1e98b Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Tue, 7 Apr 2026 15:48:12 +0200 Subject: [PATCH 09/12] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20LSP=20featur?= =?UTF-8?q?es=20howto=20guide?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- docs/howto/lsp.md | 85 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 docs/howto/lsp.md diff --git a/docs/howto/lsp.md b/docs/howto/lsp.md new file mode 100644 index 0000000000..f94c8fac27 --- /dev/null +++ b/docs/howto/lsp.md @@ -0,0 +1,85 @@ +# Language Server Protocol (LSP) support + +`elastic-package` ships a built-in LSP server for authoring integration packages. It provides real-time diagnostics, context-aware completions, and hover documentation — all driven by the official [package-spec](https://github.com/elastic/package-spec) schema and the package's own field definitions. + +## Starting the server + +```bash +elastic-package lsp +``` + +The server speaks LSP over stdin/stdout and can be wired up to any LSP-capable editor (VS Code, Neovim, Helix, etc.). + +## Features + +### Diagnostics + +The server validates the entire package against the package-spec schema as you edit. Errors are mapped to their source files and appear inline as squiggles or problem-list entries. Diagnostics clear automatically when the error is fixed. + +Validation is debounced so it doesn't fire on every keystroke — only after a short pause. + +### Completions + +**Manifest keys and values** (`manifest.yml`) + +When editing any `manifest.yml`, the server suggests valid keys at the current YAML path and enum values where applicable (e.g. `type:`, `format:`). The suggestions are schema-driven and adapt to the manifest type: integration, input, data-stream, or content. + +**Field names in pipelines** (`elasticsearch/ingest_pipeline/*.yml`) + +When the cursor is after `field:`, `target_field:`, `source:`, or `copy_to:`, the server suggests all field names defined across the package's `fields/*.yml` files. Each suggestion shows the field's type, unit, and metric type inline: + +``` +event.duration long, unit: nanos, metric: counter +http.response.bytes long, unit: byte, metric: counter +``` + +Completions are scoped to the data stream the file belongs to when possible. + +**Field types in field definitions** (`fields/*.yml`) + +When editing a `type:` key in a field definition file, the server suggests all valid Elasticsearch field types (`keyword`, `text`, `long`, `geo_point`, `nested`, etc.). + +### Hover documentation + +**Field references in pipelines** + +Hovering over a field name after `field:` / `target_field:` / etc. shows the field's full metadata: + +``` +event.duration long + +Total duration of the event in nanoseconds. + +Unit: nanos +Metric type: counter +``` + +**Manifest keys** + +Hovering over a key in `manifest.yml` shows its schema description. The server resolves the full YAML path by walking up the indentation, so nested keys get the right docs. + +**Field type / unit / metric_type values** + +Hovering over a value like `scaled_float`, `nanos`, or `counter` in a `fields/*.yml` file shows a short inline reference: + +- Field types: what the type means, storage characteristics, query limitations. +- Units: `byte`, `percent`, `ms`, `nanos`, etc. +- Metric types: `counter` vs `gauge` semantics. + +## Editor setup + +Any editor with LSP support can use this server. Point your LSP client at `elastic-package lsp` with no arguments. The server uses the `stdio` transport. + +Example VS Code `settings.json` snippet (using the generic [custom LSP extension](https://marketplace.visualstudio.com/items?itemName=genericlanguageserver.custom-lsp)): + +```json +{ + "customLsp.servers": [ + { + "name": "elastic-package", + "command": ["elastic-package", "lsp"], + "filetypes": ["yaml"] + } + ] +} +``` From fd77401c2d55ba2c070a32765658dfa0509b206a Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Tue, 7 Apr 2026 15:56:10 +0200 Subject: [PATCH 10/12] =?UTF-8?q?=F0=9F=A9=B9=20fix:=20address=20automated?= =?UTF-8?q?=20review=20findings=20in=20LSP=20code=20and=20demo=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - position.go: use len(quotedValue)-2 for JSON range end to handle escaped characters correctly - manifest_schema.go: store normalized schemaPath in valueCandidates so recursive branch calls use the resolved base path - lsp-demo.sh: fix \n in sed for BSD/macOS compatibility; fix trap quoting to defer $BROKEN_DIR expansion Co-Authored-By: Claude Sonnet 4.6 --- internal/lsp/manifest_schema.go | 2 +- internal/lsp/position.go | 2 +- scripts/lsp-demo.sh | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/lsp/manifest_schema.go b/internal/lsp/manifest_schema.go index ef73c0bf23..ff610f9543 100644 --- a/internal/lsp/manifest_schema.go +++ b/internal/lsp/manifest_schema.go @@ -383,7 +383,7 @@ func (l *manifestSchemaLoader) childProperties(schemaPath string, node map[strin } func (l *manifestSchemaLoader) valueCandidates(schemaPath string, node map[string]any) ([]string, error) { - _, current, err := l.normalize(schemaPath, node) + schemaPath, current, err := l.normalize(schemaPath, node) if err != nil { return nil, err } diff --git a/internal/lsp/position.go b/internal/lsp/position.go index f84b4e6e40..af9a8f6322 100644 --- a/internal/lsp/position.go +++ b/internal/lsp/position.go @@ -206,7 +206,7 @@ func findJSONPropertyValueRange(text, key, value string) (protocol.Range, bool) return protocol.Range{ Start: protocol.Position{Line: uint32(lineIndex), Character: uint32(start + 1)}, - End: protocol.Position{Line: uint32(lineIndex), Character: uint32(start + 1 + len(value))}, + End: protocol.Position{Line: uint32(lineIndex), Character: uint32(start + len(quotedValue) - 1)}, }, true } diff --git a/scripts/lsp-demo.sh b/scripts/lsp-demo.sh index b4a77f3ef9..f6b7cb1fcf 100755 --- a/scripts/lsp-demo.sh +++ b/scripts/lsp-demo.sh @@ -67,7 +67,8 @@ for line in sys.stdin: } extract_diags() { - sed 's/Content-Length: [0-9]*/\n---/g' | grep publishDiagnostics | print_diags + sed 's/Content-Length: [0-9]*/\ +---/g' | grep publishDiagnostics | print_diags } # ─── Scenario 1: Valid package ──────────────────────────────────────────────── @@ -91,7 +92,7 @@ fi # ─── Scenario 2: Broken package ────────────────────────────────────────────── BROKEN_DIR=$(mktemp -d) -trap "rm -rf $BROKEN_DIR" EXIT +trap 'rm -rf "$BROKEN_DIR"' EXIT mkdir -p "$BROKEN_DIR/data_stream/mystream" cat > "$BROKEN_DIR/manifest.yml" << 'YAML' From 3d794bf77038aae7c1ea333031aa2c73e896d009 Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Tue, 7 Apr 2026 16:47:31 +0200 Subject: [PATCH 11/12] =?UTF-8?q?=F0=9F=90=9B=20fix:=20infer=20indent=20co?= =?UTF-8?q?ntext=20for=20col-0=20blank=20lines=20in=20manifest=20completio?= =?UTF-8?q?ns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Editors without YAML auto-indent (e.g. Neovim) place the cursor at column 0 on a new blank line. With pos.Character=0 the line prefix was empty, yamlIndent returned 0, and path resolution fell back to root level — showing all top-level keys instead of the current block's children. Fix: when the cursor is at column 0 on a blank/whitespace-only line, infer the expected indent from the nearest preceding non-blank line. A bare key (no inline value) implies indent+2; a key-value pair implies the same indent (sibling position). Co-Authored-By: Claude Sonnet 4.6 --- internal/lsp/completion.go | 29 +++++++++++++++++++++++++++++ internal/lsp/completion_test.go | 12 ++++++++++++ 2 files changed, 41 insertions(+) diff --git a/internal/lsp/completion.go b/internal/lsp/completion.go index 47d17b46db..74a5193b99 100644 --- a/internal/lsp/completion.go +++ b/internal/lsp/completion.go @@ -283,6 +283,14 @@ func resolveManifestCompletionContext(documentText string, pos protocol.Position return manifestCompletionContext{}, false } + // When the cursor is at column 0 on a blank line (common in editors that + // don't auto-indent YAML), synthesise leading spaces from context so that + // path resolution finds the right schema node instead of falling back to + // the root level. + if pos.Character == 0 && strings.TrimSpace(lines[lineNum]) == "" { + linePrefix = inferBlankLineIndent(lines, lineNum) + } + if context, ok := resolveManifestValueContext(lines, lineNum, linePrefix); ok { return context, true } @@ -418,3 +426,24 @@ func max(a, b int) int { } return b } + +// inferBlankLineIndent returns a string of spaces representing the inferred +// indentation for a blank line. It looks backward for the nearest non-blank +// line: if that line is a bare key (no inline value), it returns indent+2 to +// represent the inside of a new block; otherwise it returns the same indent as +// the previous line (a sibling position). +func inferBlankLineIndent(lines []string, lineNum int) string { + for i := lineNum - 1; i >= 0; i-- { + line := lines[i] + if strings.TrimSpace(line) == "" { + continue + } + indent := yamlIndent(line) + _, _, hasInlineValue := yamlKeyDetails(line) + if !hasInlineValue { + return strings.Repeat(" ", indent+2) + } + return strings.Repeat(" ", indent) + } + return "" +} diff --git a/internal/lsp/completion_test.go b/internal/lsp/completion_test.go index c58e174ac9..6b2837ea38 100644 --- a/internal/lsp/completion_test.go +++ b/internal/lsp/completion_test.go @@ -169,6 +169,18 @@ streams: `) items = completeManifestItems(dataStreamManifestPath, packageRoot, documentText, pos) assert.NotNil(t, findCompletionItem(items, "enabled")) + + // Cursor at column 0 on a blank line (editors without YAML auto-indent): + // should suggest subkeys of the parent block, not root-level keys. + ownerDoc := "owner:\n" + ownerItems := completeManifestItems(manifestPath, packageRoot, ownerDoc, protocol.Position{Line: 1, Character: 0}) + assert.NotNil(t, findCompletionItem(ownerItems, "github")) + assert.Nil(t, findCompletionItem(ownerItems, "name"), "root keys must not appear inside owner block") + + conditionsDoc := "conditions:\n kibana:\n" + condItems := completeManifestItems(manifestPath, packageRoot, conditionsDoc, protocol.Position{Line: 2, Character: 0}) + assert.NotNil(t, findCompletionItem(condItems, "version")) + assert.Nil(t, findCompletionItem(condItems, "name"), "root keys must not appear inside conditions.kibana block") } func TestCompleteManifestItemsSuggestsSchemaValues(t *testing.T) { From 4d09f8beb318c01c3bbb71d3665228422d6e1fed Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Tue, 7 Apr 2026 16:54:18 +0200 Subject: [PATCH 12/12] =?UTF-8?q?=F0=9F=90=9B=20fix:=20subkey=20completion?= =?UTF-8?q?s=20for=20blank=20lines=20at=20any=20column,=20categories,=20an?= =?UTF-8?q?d=20subscription?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three related completion fixes: 1. Root keys shown even at non-zero column: linePrefixAtPosition clamps to line length, so pos.Character>0 on a truly blank line still yields linePrefix="". Widen the blank-line inference guard from pos.Character==0 to yamlIndent(linePrefix)==0 to cover this. 2. categories (and other array-of-enum fields): valueCandidates only looked at the top-level schema node. For array types the enum lives in items, not the array node itself. Add items traversal so both "- |" list-item and inline "categories: sec" positions return the full enum list. 3. When completing in list-item position and the array holds scalar values (no object keys), fall back to completeArrayEnumItems instead of returning nil. conditions.elastic.subscription values were already working; added a regression test to pin the behaviour. Co-Authored-By: Claude Sonnet 4.6 --- internal/lsp/completion.go | 37 +++++++++++++++++++++++++++------ internal/lsp/completion_test.go | 33 +++++++++++++++++++++++++---- internal/lsp/manifest_schema.go | 15 +++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/internal/lsp/completion.go b/internal/lsp/completion.go index 74a5193b99..a657fcb03c 100644 --- a/internal/lsp/completion.go +++ b/internal/lsp/completion.go @@ -166,7 +166,9 @@ func completeManifestItems(filePath, packageRoot, documentText string, pos proto func completeManifestKeyItems(kind manifestSchemaKind, context manifestCompletionContext) []protocol.CompletionItem { keys, fromArray := manifestChildKeys(context.path, kind) if len(keys) == 0 { - return nil + // The array may hold scalar enum values rather than objects (e.g. + // "categories"). Return enum values directly without a ": " suffix. + return completeArrayEnumItems(kind, context) } insertPrefix := "" @@ -198,6 +200,28 @@ func completeManifestKeyItems(kind manifestSchemaKind, context manifestCompletio return items } +// completeArrayEnumItems handles the case where the current path points to an +// array whose items are scalar enum values (e.g. "categories"). It returns the +// enum values as completion items without a ": " suffix. +func completeArrayEnumItems(kind manifestSchemaKind, context manifestCompletionContext) []protocol.CompletionItem { + values := manifestValueCandidates(context.path, kind) + if len(values) == 0 { + return nil + } + enumKind := protocol.CompletionItemKindEnumMember + var items []protocol.CompletionItem + for _, value := range values { + if context.prefix != "" && !strings.HasPrefix(value, context.prefix) { + continue + } + items = append(items, protocol.CompletionItem{ + Label: value, + Kind: &enumKind, + }) + } + return items +} + func completeManifestValueItems(kind manifestSchemaKind, context manifestCompletionContext) []protocol.CompletionItem { values := manifestValueCandidates(context.path, kind) if len(values) == 0 { @@ -283,11 +307,12 @@ func resolveManifestCompletionContext(documentText string, pos protocol.Position return manifestCompletionContext{}, false } - // When the cursor is at column 0 on a blank line (common in editors that - // don't auto-indent YAML), synthesise leading spaces from context so that - // path resolution finds the right schema node instead of falling back to - // the root level. - if pos.Character == 0 && strings.TrimSpace(lines[lineNum]) == "" { + // When the effective indent is zero on a blank line, the cursor may be + // past the end of the (empty) line — linePrefixAtPosition clamps to the + // line length, so pos.Character > 0 still yields linePrefix "". In both + // cases infer the indent from the surrounding context so that path + // resolution finds the right schema node instead of falling back to root. + if strings.TrimSpace(lines[lineNum]) == "" && yamlIndent(linePrefix) == 0 { linePrefix = inferBlankLineIndent(lines, lineNum) } diff --git a/internal/lsp/completion_test.go b/internal/lsp/completion_test.go index 6b2837ea38..48f6eb4ea9 100644 --- a/internal/lsp/completion_test.go +++ b/internal/lsp/completion_test.go @@ -172,10 +172,13 @@ streams: // Cursor at column 0 on a blank line (editors without YAML auto-indent): // should suggest subkeys of the parent block, not root-level keys. - ownerDoc := "owner:\n" - ownerItems := completeManifestItems(manifestPath, packageRoot, ownerDoc, protocol.Position{Line: 1, Character: 0}) - assert.NotNil(t, findCompletionItem(ownerItems, "github")) - assert.Nil(t, findCompletionItem(ownerItems, "name"), "root keys must not appear inside owner block") + // Also covers the case where the document has an empty line but the editor + // sends pos.Character > 0 (cursor past end-of-line). + for _, char := range []uint32{0, 2, 5} { + ownerItems := completeManifestItems(manifestPath, packageRoot, "owner:\n", protocol.Position{Line: 1, Character: char}) + assert.NotNil(t, findCompletionItem(ownerItems, "github"), "char=%d: expected owner subkey", char) + assert.Nil(t, findCompletionItem(ownerItems, "name"), "char=%d: root key must not appear inside owner block", char) + } conditionsDoc := "conditions:\n kibana:\n" condItems := completeManifestItems(manifestPath, packageRoot, conditionsDoc, protocol.Position{Line: 2, Character: 0}) @@ -225,6 +228,28 @@ streams: `) items = completeManifestItems(dataStreamManifestPath, packageRoot, documentText, pos) assert.NotNil(t, findCompletionItem(items, "true")) + + // categories: array-of-enum — completions on both "- |" list item and inline value + documentText, pos = completionDocument(t, `categories: + - | +`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "security")) + assert.NotNil(t, findCompletionItem(items, "observability")) + + documentText, pos = completionDocument(t, `categories: sec|`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "security")) + + // conditions.elastic.subscription values + documentText, pos = completionDocument(t, `conditions: + elastic: + subscription: | +`) + items = completeManifestItems(manifestPath, packageRoot, documentText, pos) + assert.NotNil(t, findCompletionItem(items, "basic")) + assert.NotNil(t, findCompletionItem(items, "gold")) + assert.NotNil(t, findCompletionItem(items, "enterprise")) } func TestCompleteFieldTypeValuesSuggestsKnownTypes(t *testing.T) { diff --git a/internal/lsp/manifest_schema.go b/internal/lsp/manifest_schema.go index ff610f9543..088b5ebdef 100644 --- a/internal/lsp/manifest_schema.go +++ b/internal/lsp/manifest_schema.go @@ -424,6 +424,21 @@ func (l *manifestSchemaLoader) valueCandidates(schemaPath string, node map[strin } } + // For array types, surface enum values from the items schema so that + // both inline ("categories: sec") and list-item ("- sec") contexts + // get value suggestions. + if items, ok := current["items"]; ok { + itemValues, err := l.valueCandidates(schemaPath, asMap(items)) + if err == nil { + for _, value := range itemValues { + if _, ok := seen[value]; !ok { + seen[value] = struct{}{} + out = append(out, value) + } + } + } + } + return out, nil }