Optimize semantic validation

code/go/internal/pkgpath/cached.go

@@ -0,0 +1,96 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package pkgpath
+
+import (
+	"sync"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+)
+
+type filesEntry struct {
+	files []File
+	err   error
+}
+
+type computedEntry struct {
+	value any
+	err   error
+}
+
+// CachedFS wraps an fspath.FS with cached file access.
+// Validators receive this type instead of raw fspath.FS, ensuring all file
+// access goes through the Files() method which caches results by glob pattern.
+type CachedFS struct {
+	fs fspath.FS
+
+	filesMu    sync.Mutex
+	filesCache map[string]filesEntry
+
+	computeMu    sync.Mutex
+	computeCache map[string]computedEntry
+}
+
+// NewCachedFS creates a CachedFS wrapping the given filesystem.
+func NewCachedFS(fsys fspath.FS) *CachedFS {
+	return &CachedFS{
+		fs:           fsys,
+		filesCache:   make(map[string]filesEntry),
+		computeCache: make(map[string]computedEntry),
+	}
+}
+
+// Files finds files matching the glob pattern. Results are cached: repeated
+// calls with the same pattern return the same File instances, sharing their
+// parsed content caches.
+func (c *CachedFS) Files(glob string) ([]File, error) {
+	c.filesMu.Lock()
+	entry, ok := c.filesCache[glob]
+	c.filesMu.Unlock()
+	if ok {
+		return entry.files, entry.err
+	}
+
+	files, err := Files(c.fs, glob)
+
+	c.filesMu.Lock()
+	c.filesCache[glob] = filesEntry{files, err}
+	c.filesMu.Unlock()
+
+	return files, err
+}
+
+// Path returns a path for the given names, based on the location of the
+// underlying filesystem. Used for error messages and linked file resolution.
+func (c *CachedFS) Path(names ...string) string {
+	return c.fs.Path(names...)
+}
+
+// RawFS returns the underlying filesystem for special cases that need
+// direct fs.FS access.
+func (c *CachedFS) RawFS() fspath.FS {
+	return c.fs
+}
+
+// LoadOrStore returns the cached value for key if present. Otherwise it calls
+// compute, stores the result, and returns it. This is useful for caching
+// derived data (e.g. parsed YAML into custom structs) that cannot use the
+// generic File.Values() cache.
+func (c *CachedFS) LoadOrStore(key string, compute func() (any, error)) (any, error) {
+	c.computeMu.Lock()
+	entry, ok := c.computeCache[key]
+	c.computeMu.Unlock()
+	if ok {
+		return entry.value, entry.err
+	}
+
+	value, err := compute()
+
+	c.computeMu.Lock()
+	c.computeCache[key] = computedEntry{value, err}
+	c.computeMu.Unlock()
+
+	return value, err
+}

code/go/internal/pkgpath/files.go

@@ -9,8 +9,10 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/PaesslerAG/jsonpath"
 	"github.com/joeshaw/multierror"
@@ -20,10 +22,19 @@ import (
 	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
 )
 
+// parsedFileContent is a lazily-initialized cache of a file's parsed content.
+// Using a pointer in File allows the cache to be shared across value copies of File.
+type parsedFileContent struct {
+	once sync.Once
+	v    interface{}
+	err  error
+}
+
 // File represents a file in the package.
 type File struct {
-	fsys fspath.FS
-	path string
+	fsys   fspath.FS
+	path   string
+	parsed *parsedFileContent
 	os.FileInfo
 }
 
@@ -43,7 +54,7 @@ func Files(fsys fspath.FS, glob string) ([]File, error) {
 			continue
 		}
 
-		file := File{fsys, path, info}
+		file := File{fsys, path, &parsedFileContent{}, info}
 		files = append(files, file)
 	}
 
@@ -61,30 +72,40 @@ func (f File) Values(path string) (interface{}, error) {
 		return nil, fmt.Errorf("cannot extract values from file type = %s", fileExt)
 	}
 
-	contents, err := fs.ReadFile(f.fsys, f.path)
-	if err != nil {
-		return nil, fmt.Errorf("reading file content failed: %w", err)
-	}
-
-	var v interface{}
-	if fileExt == "yaml" || fileExt == "yml" {
-		if err := yaml.Unmarshal(contents, &v); err != nil {
-			return nil, fmt.Errorf("unmarshalling YAML file failed (path: %s): %w", f.fsys.Path(fileName), err)
+	f.parsed.once.Do(func() {
+		contents, err := fs.ReadFile(f.fsys, f.path)
+		if err != nil {
+			f.parsed.err = fmt.Errorf("reading file content failed: %w", err)
+			return
 		}
-	} else if fileExt == "json" {
-		if err := json.Unmarshal(contents, &v); err != nil {
-			return nil, fmt.Errorf("unmarshalling JSON file failed (path: %s): %w", f.fsys.Path(fileName), err)
+
+		if fileExt == "yaml" || fileExt == "yml" {
+			if err := yaml.Unmarshal(contents, &f.parsed.v); err != nil {
+				f.parsed.err = fmt.Errorf("unmarshalling YAML file failed (path: %s): %w", f.fsys.Path(fileName), err)
+			}
+		} else if fileExt == "json" {
+			if err := json.Unmarshal(contents, &f.parsed.v); err != nil {
+				f.parsed.err = fmt.Errorf("unmarshalling JSON file failed (path: %s): %w", f.fsys.Path(fileName), err)
+			}
 		}
+	})
+	if f.parsed.err != nil {
+		return nil, f.parsed.err
 	}
 
-	return jsonpath.Get(path, v)
+	return jsonpath.Get(path, f.parsed.v)
 }
 
 // Path returns the complete path to the file.
 func (f File) Path() string {
 	return f.path
 }
 
+// Name returns the base name of the file from its path in the filesystem.
+func (f File) Name() string {
+	return path.Base(f.path)
+}
+
 // ReadAll reads and returns the entire contents of the file.
 func (f File) ReadAll() ([]byte, error) {
 	return fs.ReadFile(f.fsys, f.path)

code/go/internal/validator/semantic/types.go

@@ -6,19 +6,31 @@ package semantic
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
-	"io/fs"
-	"os"
 	"path"
 	"strconv"
 
 	"gopkg.in/yaml.v3"
 
-	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/internal/pkgpath"
 	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
 )
 
+// PackageFS is the filesystem interface that validators use to access package
+// files. It is satisfied by *pkgpath.CachedFS, which caches file access.
+type PackageFS interface {
+	// Files finds files matching the glob pattern.
+	Files(glob string) ([]pkgpath.File, error)
+
+	// Path returns a path for the given names, based on the location of
+	// the underlying filesystem. Used for error messages.
+	Path(names ...string) string
+
+	// LoadOrStore returns the cached value for key, or calls compute,
+	// stores and returns the result. Useful for caching derived data.
+	LoadOrStore(key string, compute func() (any, error)) (any, error)
+}
+
 const (
 	dataStreamDir = "data_stream"
 
@@ -179,7 +191,7 @@ type pipelineFileMetadata struct {
 
 type validateFunc func(fileMetadata fieldFileMetadata, f field) specerrors.ValidationErrors
 
-func validateFields(fsys fspath.FS, validate validateFunc) specerrors.ValidationErrors {
+func validateFields(fsys PackageFS, validate validateFunc) specerrors.ValidationErrors {
 	fieldsFilesMetadata, err := listFieldsFiles(fsys)
 	if err != nil {
 		return specerrors.ValidationErrors{
@@ -223,7 +235,7 @@ func validateNestedFields(parent string, metadata fieldFileMetadata, fields fiel
 	return result
 }
 
-func listFieldsFiles(fsys fspath.FS) ([]fieldFileMetadata, error) {
+func listFieldsFiles(fsys PackageFS) ([]fieldFileMetadata, error) {
 	var fieldsFilesMetadata []fieldFileMetadata
 
 	// integration packages
@@ -296,88 +308,92 @@ func listFieldsFiles(fsys fspath.FS) ([]fieldFileMetadata, error) {
 	return fieldsFilesMetadata, nil
 }
 
-func readFieldsFolder(fsys fspath.FS, fieldsDir string) ([]string, error) {
-	var fieldsFiles []string
-	fs, err := fs.ReadDir(fsys, fieldsDir)
-	if errors.Is(err, os.ErrNotExist) {
-		return []string{}, nil
-	}
+func readFieldsFolder(fsys PackageFS, fieldsDir string) ([]string, error) {
+	entries, err := fsys.Files(fieldsDir + "/*")
 	if err != nil {
 		return nil, fmt.Errorf("can't list fields directory (path: %s): %w", fsys.Path(fieldsDir), err)
 	}
 
-	for _, f := range fs {
-		fieldsFiles = append(fieldsFiles, path.Join(fieldsDir, f.Name()))
+	var fieldsFiles []string
+	for _, f := range entries {
+		fieldsFiles = append(fieldsFiles, f.Path())
 	}
 	return fieldsFiles, nil
 }
 
-func readPipelinesFolder(fsys fspath.FS, pipelinesDir string) ([]string, error) {
-	var pipelineFiles []string
-	entries, err := fs.ReadDir(fsys, pipelinesDir)
-	if errors.Is(err, os.ErrNotExist) {
-		return nil, nil
-	}
+func readPipelinesFolder(fsys PackageFS, pipelinesDir string) ([]string, error) {
+	entries, err := fsys.Files(pipelinesDir + "/*")
 	if err != nil {
 		return nil, fmt.Errorf("can't list pipelines directory (path: %s): %w", fsys.Path(pipelinesDir), err)
 	}
 
+	var pipelineFiles []string
 	for _, v := range entries {
-		pipelineFiles = append(pipelineFiles, path.Join(pipelinesDir, v.Name()))
+		pipelineFiles = append(pipelineFiles, v.Path())
 	}
-
 	return pipelineFiles, nil
 }
 
-func unmarshalFields(fsys fspath.FS, fieldsPath string) (fields, error) {
-	content, err := fs.ReadFile(fsys, fieldsPath)
-	if err != nil {
-		return nil, fmt.Errorf("can't read file (path: %s): %w", fieldsPath, err)
-	}
+func unmarshalFields(fsys PackageFS, fieldsPath string) (fields, error) {
+	key := "unmarshalFields:" + fieldsPath
+	result, err := fsys.LoadOrStore(key, func() (any, error) {
+		files, err := fsys.Files(fieldsPath)
+		if err != nil {
+			return nil, fmt.Errorf("can't read file (path: %s): %w", fieldsPath, err)
+		}
+		if len(files) == 0 {
+			return nil, fmt.Errorf("can't read file (path: %s): file not found", fieldsPath)
+		}
 
-	var f fields
-	err = yaml.Unmarshal(content, &f)
+		content, err := files[0].ReadAll()
+		if err != nil {
+			return nil, fmt.Errorf("can't read file (path: %s): %w", fieldsPath, err)
+		}
+
+		var f fields
+		if err := yaml.Unmarshal(content, &f); err != nil {
+			return nil, fmt.Errorf("yaml.Unmarshal failed (path: %s): %w", fieldsPath, err)
+		}
+		return f, nil
+	})
 	if err != nil {
-		return nil, fmt.Errorf("yaml.Unmarshal failed (path: %s): %w", fieldsPath, err)
+		return nil, err
 	}
-	return f, nil
+	return result.(fields), nil
 }
 
-func listDataStreams(fsys fspath.FS) ([]string, error) {
-	dataStreams, err := fs.ReadDir(fsys, dataStreamDir)
-	if errors.Is(err, os.ErrNotExist) {
-		return nil, nil
-	}
+func listDataStreams(fsys PackageFS) ([]string, error) {
+	entries, err := fsys.Files(dataStreamDir + "/*")
 	if err != nil {
 		return nil, fmt.Errorf("can't list data streams directory: %w", err)
 	}
 
-	list := make([]string, len(dataStreams))
-	for i, dataStream := range dataStreams {
-		list[i] = dataStream.Name()
+	var list []string
+	for _, entry := range entries {
+		if entry.IsDir() {
+			list = append(list, entry.Name())
+		}
 	}
 	return list, nil
 }
 
-func listTransforms(fsys fspath.FS) ([]string, error) {
+func listTransforms(fsys PackageFS) ([]string, error) {
 	transformDirectory := path.Join("elasticsearch", "transform")
-	transforms, err := fs.ReadDir(fsys, transformDirectory)
-	if errors.Is(err, os.ErrNotExist) {
-		return nil, nil
-	}
+	entries, err := fsys.Files(transformDirectory + "/*")
 	if err != nil {
 		return nil, fmt.Errorf("can't list transforms directory: %w", err)
 	}
 
-	list := make([]string, len(transforms))
-	for i, transform := range transforms {
-		list[i] = transform.Name()
+	var list []string
+	for _, entry := range entries {
+		if entry.IsDir() {
+			list = append(list, entry.Name())
+		}
 	}
 	return list, nil
-
 }
 
-func listPipelineFiles(fsys fspath.FS) ([]pipelineFileMetadata, error) {
+func listPipelineFiles(fsys PackageFS) ([]pipelineFileMetadata, error) {
 	var pipelineFileMetadatas []pipelineFileMetadata
 
 	type pipelineDirMetadata struct {

code/go/internal/validator/semantic/types_test.go

@@ -15,6 +15,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/internal/pkgpath"
 )
 
 func TestListFieldsFiles(t *testing.T) {
@@ -135,7 +136,7 @@ func TestListFieldsFiles(t *testing.T) {
 			pkgRootPath := path.Join("..", "..", "..", "..", "..", "test", "packages", c.pkgName)
 
 			fsys := fspath.DirFS(pkgRootPath)
-			fieldFilesMetadata, err := listFieldsFiles(fsys)
+			fieldFilesMetadata, err := listFieldsFiles(pkgpath.NewCachedFS(fsys))
 			require.NoError(t, err)
 
 			require.Len(t, fieldFilesMetadata, len(c.expected))