diff --git a/code/go/internal/pkgpath/cached.go b/code/go/internal/pkgpath/cached.go new file mode 100644 index 000000000..617d66dd2 --- /dev/null +++ b/code/go/internal/pkgpath/cached.go @@ -0,0 +1,96 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package pkgpath + +import ( + "sync" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" +) + +type filesEntry struct { + files []File + err error +} + +type computedEntry struct { + value any + err error +} + +// CachedFS wraps an fspath.FS with cached file access. +// Validators receive this type instead of raw fspath.FS, ensuring all file +// access goes through the Files() method which caches results by glob pattern. +type CachedFS struct { + fs fspath.FS + + filesMu sync.Mutex + filesCache map[string]filesEntry + + computeMu sync.Mutex + computeCache map[string]computedEntry +} + +// NewCachedFS creates a CachedFS wrapping the given filesystem. +func NewCachedFS(fsys fspath.FS) *CachedFS { + return &CachedFS{ + fs: fsys, + filesCache: make(map[string]filesEntry), + computeCache: make(map[string]computedEntry), + } +} + +// Files finds files matching the glob pattern. Results are cached: repeated +// calls with the same pattern return the same File instances, sharing their +// parsed content caches. +func (c *CachedFS) Files(glob string) ([]File, error) { + c.filesMu.Lock() + entry, ok := c.filesCache[glob] + c.filesMu.Unlock() + if ok { + return entry.files, entry.err + } + + files, err := Files(c.fs, glob) + + c.filesMu.Lock() + c.filesCache[glob] = filesEntry{files, err} + c.filesMu.Unlock() + + return files, err +} + +// Path returns a path for the given names, based on the location of the +// underlying filesystem. Used for error messages and linked file resolution. +func (c *CachedFS) Path(names ...string) string { + return c.fs.Path(names...) +} + +// RawFS returns the underlying filesystem for special cases that need +// direct fs.FS access. +func (c *CachedFS) RawFS() fspath.FS { + return c.fs +} + +// LoadOrStore returns the cached value for key if present. Otherwise it calls +// compute, stores the result, and returns it. This is useful for caching +// derived data (e.g. parsed YAML into custom structs) that cannot use the +// generic File.Values() cache. +func (c *CachedFS) LoadOrStore(key string, compute func() (any, error)) (any, error) { + c.computeMu.Lock() + entry, ok := c.computeCache[key] + c.computeMu.Unlock() + if ok { + return entry.value, entry.err + } + + value, err := compute() + + c.computeMu.Lock() + c.computeCache[key] = computedEntry{value, err} + c.computeMu.Unlock() + + return value, err +} diff --git a/code/go/internal/pkgpath/files.go b/code/go/internal/pkgpath/files.go index 9fd116e70..c7531852e 100644 --- a/code/go/internal/pkgpath/files.go +++ b/code/go/internal/pkgpath/files.go @@ -9,8 +9,10 @@ import ( "fmt" "io/fs" "os" + "path" "path/filepath" "strings" + "sync" "github.com/PaesslerAG/jsonpath" "github.com/joeshaw/multierror" @@ -20,10 +22,19 @@ import ( "github.com/elastic/package-spec/v3/code/go/internal/fspath" ) +// parsedFileContent is a lazily-initialized cache of a file's parsed content. +// Using a pointer in File allows the cache to be shared across value copies of File. +type parsedFileContent struct { + once sync.Once + v interface{} + err error +} + // File represents a file in the package. type File struct { - fsys fspath.FS - path string + fsys fspath.FS + path string + parsed *parsedFileContent os.FileInfo } @@ -43,7 +54,7 @@ func Files(fsys fspath.FS, glob string) ([]File, error) { continue } - file := File{fsys, path, info} + file := File{fsys, path, &parsedFileContent{}, info} files = append(files, file) } @@ -61,23 +72,28 @@ func (f File) Values(path string) (interface{}, error) { return nil, fmt.Errorf("cannot extract values from file type = %s", fileExt) } - contents, err := fs.ReadFile(f.fsys, f.path) - if err != nil { - return nil, fmt.Errorf("reading file content failed: %w", err) - } - - var v interface{} - if fileExt == "yaml" || fileExt == "yml" { - if err := yaml.Unmarshal(contents, &v); err != nil { - return nil, fmt.Errorf("unmarshalling YAML file failed (path: %s): %w", f.fsys.Path(fileName), err) + f.parsed.once.Do(func() { + contents, err := fs.ReadFile(f.fsys, f.path) + if err != nil { + f.parsed.err = fmt.Errorf("reading file content failed: %w", err) + return } - } else if fileExt == "json" { - if err := json.Unmarshal(contents, &v); err != nil { - return nil, fmt.Errorf("unmarshalling JSON file failed (path: %s): %w", f.fsys.Path(fileName), err) + + if fileExt == "yaml" || fileExt == "yml" { + if err := yaml.Unmarshal(contents, &f.parsed.v); err != nil { + f.parsed.err = fmt.Errorf("unmarshalling YAML file failed (path: %s): %w", f.fsys.Path(fileName), err) + } + } else if fileExt == "json" { + if err := json.Unmarshal(contents, &f.parsed.v); err != nil { + f.parsed.err = fmt.Errorf("unmarshalling JSON file failed (path: %s): %w", f.fsys.Path(fileName), err) + } } + }) + if f.parsed.err != nil { + return nil, f.parsed.err } - return jsonpath.Get(path, v) + return jsonpath.Get(path, f.parsed.v) } // Path returns the complete path to the file. @@ -85,6 +101,11 @@ func (f File) Path() string { return f.path } +// Name returns the base name of the file from its path in the filesystem. +func (f File) Name() string { + return path.Base(f.path) +} + // ReadAll reads and returns the entire contents of the file. func (f File) ReadAll() ([]byte, error) { return fs.ReadFile(f.fsys, f.path) diff --git a/code/go/internal/validator/semantic/types.go b/code/go/internal/validator/semantic/types.go index 51a1fc5b7..47860f32d 100644 --- a/code/go/internal/validator/semantic/types.go +++ b/code/go/internal/validator/semantic/types.go @@ -6,19 +6,31 @@ package semantic import ( "encoding/json" - "errors" "fmt" - "io/fs" - "os" "path" "strconv" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) +// PackageFS is the filesystem interface that validators use to access package +// files. It is satisfied by *pkgpath.CachedFS, which caches file access. +type PackageFS interface { + // Files finds files matching the glob pattern. + Files(glob string) ([]pkgpath.File, error) + + // Path returns a path for the given names, based on the location of + // the underlying filesystem. Used for error messages. + Path(names ...string) string + + // LoadOrStore returns the cached value for key, or calls compute, + // stores and returns the result. Useful for caching derived data. + LoadOrStore(key string, compute func() (any, error)) (any, error) +} + const ( dataStreamDir = "data_stream" @@ -179,7 +191,7 @@ type pipelineFileMetadata struct { type validateFunc func(fileMetadata fieldFileMetadata, f field) specerrors.ValidationErrors -func validateFields(fsys fspath.FS, validate validateFunc) specerrors.ValidationErrors { +func validateFields(fsys PackageFS, validate validateFunc) specerrors.ValidationErrors { fieldsFilesMetadata, err := listFieldsFiles(fsys) if err != nil { return specerrors.ValidationErrors{ @@ -223,7 +235,7 @@ func validateNestedFields(parent string, metadata fieldFileMetadata, fields fiel return result } -func listFieldsFiles(fsys fspath.FS) ([]fieldFileMetadata, error) { +func listFieldsFiles(fsys PackageFS) ([]fieldFileMetadata, error) { var fieldsFilesMetadata []fieldFileMetadata // integration packages @@ -296,88 +308,92 @@ func listFieldsFiles(fsys fspath.FS) ([]fieldFileMetadata, error) { return fieldsFilesMetadata, nil } -func readFieldsFolder(fsys fspath.FS, fieldsDir string) ([]string, error) { - var fieldsFiles []string - fs, err := fs.ReadDir(fsys, fieldsDir) - if errors.Is(err, os.ErrNotExist) { - return []string{}, nil - } +func readFieldsFolder(fsys PackageFS, fieldsDir string) ([]string, error) { + entries, err := fsys.Files(fieldsDir + "/*") if err != nil { return nil, fmt.Errorf("can't list fields directory (path: %s): %w", fsys.Path(fieldsDir), err) } - for _, f := range fs { - fieldsFiles = append(fieldsFiles, path.Join(fieldsDir, f.Name())) + var fieldsFiles []string + for _, f := range entries { + fieldsFiles = append(fieldsFiles, f.Path()) } return fieldsFiles, nil } -func readPipelinesFolder(fsys fspath.FS, pipelinesDir string) ([]string, error) { - var pipelineFiles []string - entries, err := fs.ReadDir(fsys, pipelinesDir) - if errors.Is(err, os.ErrNotExist) { - return nil, nil - } +func readPipelinesFolder(fsys PackageFS, pipelinesDir string) ([]string, error) { + entries, err := fsys.Files(pipelinesDir + "/*") if err != nil { return nil, fmt.Errorf("can't list pipelines directory (path: %s): %w", fsys.Path(pipelinesDir), err) } + var pipelineFiles []string for _, v := range entries { - pipelineFiles = append(pipelineFiles, path.Join(pipelinesDir, v.Name())) + pipelineFiles = append(pipelineFiles, v.Path()) } - return pipelineFiles, nil } -func unmarshalFields(fsys fspath.FS, fieldsPath string) (fields, error) { - content, err := fs.ReadFile(fsys, fieldsPath) - if err != nil { - return nil, fmt.Errorf("can't read file (path: %s): %w", fieldsPath, err) - } +func unmarshalFields(fsys PackageFS, fieldsPath string) (fields, error) { + key := "unmarshalFields:" + fieldsPath + result, err := fsys.LoadOrStore(key, func() (any, error) { + files, err := fsys.Files(fieldsPath) + if err != nil { + return nil, fmt.Errorf("can't read file (path: %s): %w", fieldsPath, err) + } + if len(files) == 0 { + return nil, fmt.Errorf("can't read file (path: %s): file not found", fieldsPath) + } - var f fields - err = yaml.Unmarshal(content, &f) + content, err := files[0].ReadAll() + if err != nil { + return nil, fmt.Errorf("can't read file (path: %s): %w", fieldsPath, err) + } + + var f fields + if err := yaml.Unmarshal(content, &f); err != nil { + return nil, fmt.Errorf("yaml.Unmarshal failed (path: %s): %w", fieldsPath, err) + } + return f, nil + }) if err != nil { - return nil, fmt.Errorf("yaml.Unmarshal failed (path: %s): %w", fieldsPath, err) + return nil, err } - return f, nil + return result.(fields), nil } -func listDataStreams(fsys fspath.FS) ([]string, error) { - dataStreams, err := fs.ReadDir(fsys, dataStreamDir) - if errors.Is(err, os.ErrNotExist) { - return nil, nil - } +func listDataStreams(fsys PackageFS) ([]string, error) { + entries, err := fsys.Files(dataStreamDir + "/*") if err != nil { return nil, fmt.Errorf("can't list data streams directory: %w", err) } - list := make([]string, len(dataStreams)) - for i, dataStream := range dataStreams { - list[i] = dataStream.Name() + var list []string + for _, entry := range entries { + if entry.IsDir() { + list = append(list, entry.Name()) + } } return list, nil } -func listTransforms(fsys fspath.FS) ([]string, error) { +func listTransforms(fsys PackageFS) ([]string, error) { transformDirectory := path.Join("elasticsearch", "transform") - transforms, err := fs.ReadDir(fsys, transformDirectory) - if errors.Is(err, os.ErrNotExist) { - return nil, nil - } + entries, err := fsys.Files(transformDirectory + "/*") if err != nil { return nil, fmt.Errorf("can't list transforms directory: %w", err) } - list := make([]string, len(transforms)) - for i, transform := range transforms { - list[i] = transform.Name() + var list []string + for _, entry := range entries { + if entry.IsDir() { + list = append(list, entry.Name()) + } } return list, nil - } -func listPipelineFiles(fsys fspath.FS) ([]pipelineFileMetadata, error) { +func listPipelineFiles(fsys PackageFS) ([]pipelineFileMetadata, error) { var pipelineFileMetadatas []pipelineFileMetadata type pipelineDirMetadata struct { diff --git a/code/go/internal/validator/semantic/types_test.go b/code/go/internal/validator/semantic/types_test.go index 5897ded07..ca698ddd3 100644 --- a/code/go/internal/validator/semantic/types_test.go +++ b/code/go/internal/validator/semantic/types_test.go @@ -15,6 +15,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestListFieldsFiles(t *testing.T) { @@ -135,7 +136,7 @@ func TestListFieldsFiles(t *testing.T) { pkgRootPath := path.Join("..", "..", "..", "..", "..", "test", "packages", c.pkgName) fsys := fspath.DirFS(pkgRootPath) - fieldFilesMetadata, err := listFieldsFiles(fsys) + fieldFilesMetadata, err := listFieldsFiles(pkgpath.NewCachedFS(fsys)) require.NoError(t, err) require.Len(t, fieldFilesMetadata, len(c.expected)) diff --git a/code/go/internal/validator/semantic/validate_agent_version.go b/code/go/internal/validator/semantic/validate_agent_version.go index 53738d38e..fcc94366d 100644 --- a/code/go/internal/validator/semantic/validate_agent_version.go +++ b/code/go/internal/validator/semantic/validate_agent_version.go @@ -9,7 +9,6 @@ import ( "github.com/Masterminds/semver/v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -20,7 +19,7 @@ var ( ) // ValidateMinimumAgentVersion checks that the package manifest includes the agent.version condition. -func ValidateMinimumAgentVersion(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateMinimumAgentVersion(fsys PackageFS) specerrors.ValidationErrors { manifest, err := readManifest(fsys) if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} diff --git a/code/go/internal/validator/semantic/validate_agent_version_test.go b/code/go/internal/validator/semantic/validate_agent_version_test.go index 1276bde20..cda605bd8 100644 --- a/code/go/internal/validator/semantic/validate_agent_version_test.go +++ b/code/go/internal/validator/semantic/validate_agent_version_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateMinimumAgentVersion(t *testing.T) { @@ -73,7 +74,7 @@ conditions: require.NoError(t, err) fsys := fspath.DirFS(tempDir) - errs := ValidateMinimumAgentVersion(fsys) + errs := ValidateMinimumAgentVersion(pkgpath.NewCachedFS(fsys)) if c.expectedErr != nil { require.Len(t, errs, 1) diff --git a/code/go/internal/validator/semantic/validate_capabilities_required.go b/code/go/internal/validator/semantic/validate_capabilities_required.go index fe2bc35e7..aa8b884f8 100644 --- a/code/go/internal/validator/semantic/validate_capabilities_required.go +++ b/code/go/internal/validator/semantic/validate_capabilities_required.go @@ -9,13 +9,11 @@ import ( "path" "slices" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" - "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateCapabilitiesRequired verifies that the required capabilities are added in package manifest -func ValidateCapabilitiesRequired(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateCapabilitiesRequired(fsys PackageFS) specerrors.ValidationErrors { err := ensureSecurityRulesHasSecurityCapability(fsys) if err != nil { return err @@ -23,9 +21,9 @@ func ValidateCapabilitiesRequired(fsys fspath.FS) specerrors.ValidationErrors { return nil } -func ensureSecurityRulesHasSecurityCapability(fsys fspath.FS) specerrors.ValidationErrors { +func ensureSecurityRulesHasSecurityCapability(fsys PackageFS) specerrors.ValidationErrors { securityRuleFilePaths := path.Join("kibana", "security_rule", "*.json") - files, err := pkgpath.Files(fsys, securityRuleFilePaths) + files, err := fsys.Files(securityRuleFilePaths) if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("error finding Kibana security_rule folder: %w", err)} } @@ -47,7 +45,7 @@ func ensureSecurityRulesHasSecurityCapability(fsys fspath.FS) specerrors.Validat return nil } -func readCapabilities(fsys fspath.FS) ([]string, error) { +func readCapabilities(fsys PackageFS) ([]string, error) { manifest, err := readManifest(fsys) if err != nil { return nil, err diff --git a/code/go/internal/validator/semantic/validate_changelog_links.go b/code/go/internal/validator/semantic/validate_changelog_links.go index 39612329f..d7e73b660 100644 --- a/code/go/internal/validator/semantic/validate_changelog_links.go +++ b/code/go/internal/validator/semantic/validate_changelog_links.go @@ -12,7 +12,6 @@ import ( "strconv" "strings" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -36,7 +35,7 @@ func (e ChangelogLinkError) Error() string { // ValidateChangelogLinks returns validation errors if the link(s) do not have a valid PR github.com link. // If the link is not a github.com link this validation is skipped and does not return an error. -func ValidateChangelogLinks(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateChangelogLinks(fsys PackageFS) specerrors.ValidationErrors { changelogLinks, err := readChangelogLinks(fsys) if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} @@ -44,7 +43,7 @@ func ValidateChangelogLinks(fsys fspath.FS) specerrors.ValidationErrors { return ensureLinksAreValid(changelogLinks) } -func readChangelogLinks(fsys fspath.FS) ([]string, error) { +func readChangelogLinks(fsys PackageFS) ([]string, error) { return readChangelog(fsys, `$[*].changes[*].link`) } diff --git a/code/go/internal/validator/semantic/validate_date.go b/code/go/internal/validator/semantic/validate_date.go index 8b639adf5..8f1b535c6 100644 --- a/code/go/internal/validator/semantic/validate_date.go +++ b/code/go/internal/validator/semantic/validate_date.go @@ -5,12 +5,11 @@ package semantic import ( - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateDateFields verifies if date fields are of one of the expected types. -func ValidateDateFields(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateDateFields(fsys PackageFS) specerrors.ValidationErrors { return validateFields(fsys, validateDateField) } diff --git a/code/go/internal/validator/semantic/validate_deployment_modes.go b/code/go/internal/validator/semantic/validate_deployment_modes.go index 1245e55b2..6b911d624 100644 --- a/code/go/internal/validator/semantic/validate_deployment_modes.go +++ b/code/go/internal/validator/semantic/validate_deployment_modes.go @@ -6,19 +6,24 @@ package semantic import ( "fmt" - "io/fs" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateDeploymentModes ensures that for each deployment mode enabled in a policy template, // there is at least one input that supports that deployment mode. -func ValidateDeploymentModes(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateDeploymentModes(fsys PackageFS) specerrors.ValidationErrors { manifestPath := "manifest.yml" - d, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(manifestPath), err)} + } + if len(files) == 0 { + return nil + } + d, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(manifestPath), err)} } diff --git a/code/go/internal/validator/semantic/validate_deployment_modes_test.go b/code/go/internal/validator/semantic/validate_deployment_modes_test.go index 1e60ac962..353d7a809 100644 --- a/code/go/internal/validator/semantic/validate_deployment_modes_test.go +++ b/code/go/internal/validator/semantic/validate_deployment_modes_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateDeploymentModes(t *testing.T) { @@ -201,7 +202,7 @@ policy_templates: require.NoError(t, err) fsys := fspath.DirFS(tempDir) - errs := ValidateDeploymentModes(fsys) + errs := ValidateDeploymentModes(pkgpath.NewCachedFS(fsys)) if len(c.expectedErrs) == 0 { assert.Empty(t, errs) diff --git a/code/go/internal/validator/semantic/validate_deprecated_replaced_by.go b/code/go/internal/validator/semantic/validate_deprecated_replaced_by.go index 439c39352..b5d1d5741 100644 --- a/code/go/internal/validator/semantic/validate_deprecated_replaced_by.go +++ b/code/go/internal/validator/semantic/validate_deprecated_replaced_by.go @@ -5,11 +5,8 @@ package semantic import ( - "io/fs" - "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -26,14 +23,14 @@ type deprecatedInfo struct { } // ValidateDeprecatedReplacedBy checks that when deprecated.replaced_by is used, the required fields are set. -func ValidateDeprecatedReplacedBy(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateDeprecatedReplacedBy(fsys PackageFS) specerrors.ValidationErrors { errs := validatePackageManifestDeprecatedReplacedBy(fsys) dsErrs := validateDataStreamsDeprecatedReplacedBy(fsys) return append(errs, dsErrs...) } -func validatePackageManifestDeprecatedReplacedBy(fsys fspath.FS) specerrors.ValidationErrors { +func validatePackageManifestDeprecatedReplacedBy(fsys PackageFS) specerrors.ValidationErrors { // package manifest structure type manifest struct { Type string `yaml:"type,omitempty"` @@ -47,7 +44,15 @@ func validatePackageManifestDeprecatedReplacedBy(fsys fspath.FS) specerrors.Vali } manifestPath := "manifest.yml" - data, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} + } + if len(files) == 0 { + return nil + } + data, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} @@ -86,7 +91,7 @@ func validatePackageManifestDeprecatedReplacedBy(fsys fspath.FS) specerrors.Vali } -func validateDataStreamsDeprecatedReplacedBy(fsys fspath.FS) specerrors.ValidationErrors { +func validateDataStreamsDeprecatedReplacedBy(fsys PackageFS) specerrors.ValidationErrors { // stream manifest structure type streamManifest struct { Deprecated *deprecatedInfo `yaml:"deprecated,omitempty"` @@ -97,29 +102,29 @@ func validateDataStreamsDeprecatedReplacedBy(fsys fspath.FS) specerrors.Validati } `yaml:"streams,omitempty"` } - dsManifests, err := fs.Glob(fsys, "data_stream/*/manifest.yml") + dsManifestFiles, err := fsys.Files("data_stream/*/manifest.yml") if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("error while searching for data stream manifests: %w", err)} } var errs specerrors.ValidationErrors - for _, dsManifestPath := range dsManifests { - data, err := fs.ReadFile(fsys, dsManifestPath) + for _, dsManifestFile := range dsManifestFiles { + data, err := dsManifestFile.ReadAll() if err != nil { - errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(dsManifestPath), err)) + errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(dsManifestFile.Path()), err)) continue } var sm streamManifest err = yaml.Unmarshal(data, &sm) if err != nil { - errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(dsManifestPath), err)) + errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(dsManifestFile.Path()), err)) continue } if sm.Deprecated != nil && sm.Deprecated.ReplacedBy != nil { rb := sm.Deprecated.ReplacedBy if rb.DataStream == "" { - errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: deprecated.replaced_by.data_stream must be specified when deprecated.replaced_by is used", fsys.Path(dsManifestPath))) + errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: deprecated.replaced_by.data_stream must be specified when deprecated.replaced_by is used", fsys.Path(dsManifestFile.Path()))) } } @@ -128,7 +133,7 @@ func validateDataStreamsDeprecatedReplacedBy(fsys fspath.FS) specerrors.Validati if v.Deprecated != nil && v.Deprecated.ReplacedBy != nil { rb := v.Deprecated.ReplacedBy if rb.Variable == "" { - errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: variable deprecated.replaced_by.variable must be specified when deprecated.replaced_by is used", fsys.Path(dsManifestPath))) + errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: variable deprecated.replaced_by.variable must be specified when deprecated.replaced_by is used", fsys.Path(dsManifestFile.Path()))) } } } diff --git a/code/go/internal/validator/semantic/validate_deprecated_replaced_by_test.go b/code/go/internal/validator/semantic/validate_deprecated_replaced_by_test.go index 42ecb3af8..5d9652d88 100644 --- a/code/go/internal/validator/semantic/validate_deprecated_replaced_by_test.go +++ b/code/go/internal/validator/semantic/validate_deprecated_replaced_by_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidatePackageManifestDeprecatedReplacedBy(t *testing.T) { @@ -29,7 +30,7 @@ deprecated: `), 0o644) require.NoError(t, err) - errs := validatePackageManifestDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validatePackageManifestDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -48,7 +49,7 @@ deprecated: `), 0o644) require.NoError(t, err) - errs := validatePackageManifestDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validatePackageManifestDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, "deprecated.replaced_by.package must be specified when deprecated.replaced_by is used") @@ -69,7 +70,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := validatePackageManifestDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validatePackageManifestDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -88,7 +89,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := validatePackageManifestDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validatePackageManifestDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, "policy_template deprecated.replaced_by.policy_template must be specified when deprecated.replaced_by is used") @@ -111,7 +112,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := validatePackageManifestDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validatePackageManifestDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -132,7 +133,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := validatePackageManifestDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validatePackageManifestDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, "input deprecated.replaced_by.input must be specified when deprecated.replaced_by is used") @@ -152,7 +153,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := validatePackageManifestDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validatePackageManifestDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -170,7 +171,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := validatePackageManifestDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validatePackageManifestDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, "policy_template deprecated.replaced_by.policy_template must be specified when deprecated.replaced_by is used") @@ -196,7 +197,7 @@ streams: `), 0o644) require.NoError(t, err) - errs := validateDataStreamsDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validateDataStreamsDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -217,7 +218,7 @@ streams: `), 0o644) require.NoError(t, err) - errs := validateDataStreamsDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validateDataStreamsDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, "deprecated.replaced_by.data_stream must be specified when deprecated.replaced_by is used") @@ -239,7 +240,7 @@ streams: `), 0o644) require.NoError(t, err) - errs := validateDataStreamsDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validateDataStreamsDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -260,7 +261,7 @@ streams: `), 0o644) require.NoError(t, err) - errs := validateDataStreamsDeprecatedReplacedBy(fspath.DirFS(d)) + errs := validateDataStreamsDeprecatedReplacedBy(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, "variable deprecated.replaced_by.variable must be specified when deprecated.replaced_by is used") diff --git a/code/go/internal/validator/semantic/validate_dimensions.go b/code/go/internal/validator/semantic/validate_dimensions.go index 9c5cc5ae1..6ba4fc05b 100644 --- a/code/go/internal/validator/semantic/validate_dimensions.go +++ b/code/go/internal/validator/semantic/validate_dimensions.go @@ -7,12 +7,11 @@ package semantic import ( "strings" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateDimensionFields verifies if dimension fields are of one of the expected types. -func ValidateDimensionFields(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateDimensionFields(fsys PackageFS) specerrors.ValidationErrors { return validateFields(fsys, validateDimensionField) } diff --git a/code/go/internal/validator/semantic/validate_dimensions_present.go b/code/go/internal/validator/semantic/validate_dimensions_present.go index 65b79aa04..059ff8967 100644 --- a/code/go/internal/validator/semantic/validate_dimensions_present.go +++ b/code/go/internal/validator/semantic/validate_dimensions_present.go @@ -6,17 +6,15 @@ package semantic import ( "fmt" - "io/fs" "path" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateDimensionsPresent verifies if dimension fields are of one of the expected types. -func ValidateDimensionsPresent(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateDimensionsPresent(fsys PackageFS) specerrors.ValidationErrors { dimensionPresent := make(map[string]struct{}) errs := validateFields(fsys, func(metadata fieldFileMetadata, f field) specerrors.ValidationErrors { if f.Dimension { @@ -48,9 +46,16 @@ func ValidateDimensionsPresent(fsys fspath.FS) specerrors.ValidationErrors { return errs } -func isTimeSeriesModeEnabled(fsys fspath.FS, dataStream string) (bool, error) { +func isTimeSeriesModeEnabled(fsys PackageFS, dataStream string) (bool, error) { manifestPath := path.Join("data_stream", dataStream, "manifest.yml") - d, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return false, fmt.Errorf("failed to read data stream manifest in %q: %w", fsys.Path(manifestPath), err) + } + if len(files) == 0 { + return false, fmt.Errorf("failed to read data stream manifest in %q: file not found", fsys.Path(manifestPath)) + } + d, err := files[0].ReadAll() if err != nil { return false, fmt.Errorf("failed to read data stream manifest in %q: %w", fsys.Path(manifestPath), err) } diff --git a/code/go/internal/validator/semantic/validate_docs_structure.go b/code/go/internal/validator/semantic/validate_docs_structure.go index 861fa8738..9d4689802 100644 --- a/code/go/internal/validator/semantic/validate_docs_structure.go +++ b/code/go/internal/validator/semantic/validate_docs_structure.go @@ -18,8 +18,6 @@ import ( "gopkg.in/yaml.v3" spec "github.com/elastic/package-spec/v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" - "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -35,7 +33,7 @@ type Section struct { } // ValidateDocsStructure validates the structure of documentation files against enforced sections. -func ValidateDocsStructure(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateDocsStructure(fsys PackageFS) specerrors.ValidationErrors { config, err := shouldValidateDocsStructure(fsys) if err != nil { return specerrors.ValidationErrors{ @@ -72,9 +70,9 @@ func ValidateDocsStructure(fsys fspath.FS) specerrors.ValidationErrors { return nil } -func shouldValidateDocsStructure(fsys fspath.FS) (*specerrors.DocsStructureEnforced, error) { +func shouldValidateDocsStructure(fsys PackageFS) (*specerrors.DocsStructureEnforced, error) { validationPath := "validation.yml" - files, err := pkgpath.Files(fsys, validationPath) + files, err := fsys.Files(validationPath) if err != nil || len(files) == 0 { return nil, nil } @@ -97,7 +95,7 @@ func shouldValidateDocsStructure(fsys fspath.FS) (*specerrors.DocsStructureEnfor return nil, nil } -func readDocsStructureEnforcedConfig(fsys fspath.FS, config *specerrors.DocsStructureEnforced) ([]string, error) { +func readDocsStructureEnforcedConfig(fsys PackageFS, config *specerrors.DocsStructureEnforced) ([]string, error) { defaultSections, err := loadSectionsFromConfig(fmt.Sprintf("%d", config.Version)) if err != nil { return nil, fmt.Errorf("failed to load enforced sections from config: %w", err) @@ -139,9 +137,9 @@ func loadSectionsFromConfig(version string) ([]string, error) { return sections, nil } -func validateReadmeStructure(fsys fspath.FS, enforcedSections []string) error { +func validateReadmeStructure(fsys PackageFS, enforcedSections []string) error { var errs []error - files, err := pkgpath.Files(fsys, "docs/*.md") + files, err := fsys.Files("docs/*.md") if err != nil { return fmt.Errorf("docs folder %s not found: %w", "docs/*.md", err) } diff --git a/code/go/internal/validator/semantic/validate_docs_structure_test.go b/code/go/internal/validator/semantic/validate_docs_structure_test.go index aa58411d8..2b3cdba99 100644 --- a/code/go/internal/validator/semantic/validate_docs_structure_test.go +++ b/code/go/internal/validator/semantic/validate_docs_structure_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -75,7 +76,7 @@ func TestReadValidateDocsStructureConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actualResult, err := readDocsStructureEnforcedConfig(fspath.DirFS(tt.pkgRoot), tt.config) + actualResult, err := readDocsStructureEnforcedConfig(pkgpath.NewCachedFS(fspath.DirFS(tt.pkgRoot)), tt.config) assert.Equal(t, tt.expectedResult, actualResult, "Result does not match expected") assert.Equal(t, tt.expectedError, err, "Error does not match expected") }) @@ -113,7 +114,7 @@ func TestValidateDocsStructure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := ValidateDocsStructure(fspath.DirFS(tt.pkgRoot)) + err := ValidateDocsStructure(pkgpath.NewCachedFS(fspath.DirFS(tt.pkgRoot))) assert.True(t, compareErrors(tt.expectError, tt.expectedError, err), "Error does not match expected") }) } diff --git a/code/go/internal/validator/semantic/validate_duration_variables.go b/code/go/internal/validator/semantic/validate_duration_variables.go index 7b9a56e66..06d9c5402 100644 --- a/code/go/internal/validator/semantic/validate_duration_variables.go +++ b/code/go/internal/validator/semantic/validate_duration_variables.go @@ -7,13 +7,11 @@ package semantic import ( "errors" "fmt" - "io/fs" "reflect" "time" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -23,9 +21,16 @@ import ( // // It examines both the root manifest.yml file and all data stream manifests // to find and validate duration variables. -func ValidateDurationVariables(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateDurationVariables(fsys PackageFS) specerrors.ValidationErrors { // Load main manifest vars. - data, err := fs.ReadFile(fsys, "manifest.yml") + manifestFiles, err := fsys.Files("manifest.yml") + if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("%s failed to read manifest: %w", fsys.Path("manifest.yml"), err)} + } + if len(manifestFiles) == 0 { + return nil + } + data, err := manifestFiles[0].ReadAll() if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("%s failed to read manifest: %w", fsys.Path("manifest.yml"), err)} } @@ -38,25 +43,25 @@ func ValidateDurationVariables(fsys fspath.FS) specerrors.ValidationErrors { annotateFileMetadata(fsys.Path("manifest.yml"), manifest) vars := manifest.allVars() - dsManifests, err := fs.Glob(fsys, "data_stream/*/manifest.yml") + dsManifestFiles, err := fsys.Files("data_stream/*/manifest.yml") if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("failed to list data streams: %w", err)} } // Load data stream manifest vars. - for _, path := range dsManifests { - data, err := fs.ReadFile(fsys, path) + for _, dsFile := range dsManifestFiles { + data, err := dsFile.ReadAll() if err != nil { - return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("%s failed to read data stream manifest: %w", fsys.Path(path), err)} + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("%s failed to read data stream manifest: %w", fsys.Path(dsFile.Path()), err)} } - var manifest durationDataStreamManifest - err = yaml.Unmarshal(data, &manifest) + var dsManifest durationDataStreamManifest + err = yaml.Unmarshal(data, &dsManifest) if err != nil { - return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("%s is invalid: failed to parse data stream manifest: %w", fsys.Path(path), err)} + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("%s is invalid: failed to parse data stream manifest: %w", fsys.Path(dsFile.Path()), err)} } - annotateFileMetadata(fsys.Path(path), manifest) - vars = append(vars, manifest.allVars()...) + annotateFileMetadata(fsys.Path(dsFile.Path()), dsManifest) + vars = append(vars, dsManifest.allVars()...) } // Validate duration vars. diff --git a/code/go/internal/validator/semantic/validate_duration_variables_test.go b/code/go/internal/validator/semantic/validate_duration_variables_test.go index b448f4e70..c96abff64 100644 --- a/code/go/internal/validator/semantic/validate_duration_variables_test.go +++ b/code/go/internal/validator/semantic/validate_duration_variables_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateDurationVar(t *testing.T) { @@ -303,7 +304,7 @@ streams: filepath.Join("data_stream", "foo", "manifest.yml") + `:8:9 error in variable "dwell_time": min_duration "50ms50ms" greater than default "5ms"`, } - errs := ValidateDurationVariables(fspath.DirFS(d)) + errs := ValidateDurationVariables(pkgpath.NewCachedFS(fspath.DirFS(d))) if len(errs) != len(want) { t.Fatalf("Expected %d errors, got %d", len(want), len(errs)) } diff --git a/code/go/internal/validator/semantic/validate_external_fields_with_dev_folder.go b/code/go/internal/validator/semantic/validate_external_fields_with_dev_folder.go index 37a43d08b..8d5a4cade 100644 --- a/code/go/internal/validator/semantic/validate_external_fields_with_dev_folder.go +++ b/code/go/internal/validator/semantic/validate_external_fields_with_dev_folder.go @@ -7,17 +7,16 @@ package semantic import ( "fmt" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateExternalFieldsWithDevFolder verifies there is no field with external key if there is no _dev/build/build.yml definition -func ValidateExternalFieldsWithDevFolder(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateExternalFieldsWithDevFolder(fsys PackageFS) specerrors.ValidationErrors { const buildPath = "_dev/build/build.yml" buildFilePathDefined := true - f, err := pkgpath.Files(fsys, buildPath) + f, err := fsys.Files(buildPath) if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("not able to read _dev/build/build.yml: %w", err), diff --git a/code/go/internal/validator/semantic/validate_field_groups.go b/code/go/internal/validator/semantic/validate_field_groups.go index e6413bed6..14b1e81de 100644 --- a/code/go/internal/validator/semantic/validate_field_groups.go +++ b/code/go/internal/validator/semantic/validate_field_groups.go @@ -5,12 +5,11 @@ package semantic import ( - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateFieldGroups verifies if field groups don't have units and metric types defined. -func ValidateFieldGroups(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateFieldGroups(fsys PackageFS) specerrors.ValidationErrors { return validateFields(fsys, validateFieldUnit) } diff --git a/code/go/internal/validator/semantic/validate_field_groups_test.go b/code/go/internal/validator/semantic/validate_field_groups_test.go index 9f51d390b..806716b03 100644 --- a/code/go/internal/validator/semantic/validate_field_groups_test.go +++ b/code/go/internal/validator/semantic/validate_field_groups_test.go @@ -13,12 +13,13 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateFieldGroups_Good(t *testing.T) { pkgRoot := filepath.Join("..", "..", "..", "..", "..", "test", "packages", "good") - errs := ValidateFieldGroups(fspath.DirFS(pkgRoot)) + errs := ValidateFieldGroups(pkgpath.NewCachedFS(fspath.DirFS(pkgRoot))) require.Empty(t, errs) } @@ -31,7 +32,7 @@ func TestValidateFieldGroups_Bad(t *testing.T) { expected) } - errs := ValidateFieldGroups(fspath.DirFS(pkgRoot)) + errs := ValidateFieldGroups(pkgpath.NewCachedFS(fspath.DirFS(pkgRoot))) if assert.Len(t, errs, 3) { assert.Equal(t, fileError(filepath.Join("data_stream", "bar", "fields", "hello-world.yml"), `field "aaa.bbb" can't have unit property'`), errs[0].Error()) assert.Equal(t, fileError(filepath.Join("data_stream", "bar", "fields", "hello-world.yml"), `field "ddd.eee" can't have unit property'`), errs[1].Error()) diff --git a/code/go/internal/validator/semantic/validate_fields_limits.go b/code/go/internal/validator/semantic/validate_fields_limits.go index 3ba662998..8d06191e8 100644 --- a/code/go/internal/validator/semantic/validate_fields_limits.go +++ b/code/go/internal/validator/semantic/validate_fields_limits.go @@ -5,18 +5,17 @@ package semantic import ( - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateFieldsLimits verifies limits on fields. -func ValidateFieldsLimits(limit int) func(fspath.FS) specerrors.ValidationErrors { - return func(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateFieldsLimits(limit int) func(PackageFS) specerrors.ValidationErrors { + return func(fsys PackageFS) specerrors.ValidationErrors { return validateFieldsLimits(fsys, limit) } } -func validateFieldsLimits(fsys fspath.FS, limit int) specerrors.ValidationErrors { +func validateFieldsLimits(fsys PackageFS, limit int) specerrors.ValidationErrors { counts := make(map[string]int) // Created a new map to avoid collisions with data stream names transformCounts := make(map[string]int) diff --git a/code/go/internal/validator/semantic/validate_fields_limits_test.go b/code/go/internal/validator/semantic/validate_fields_limits_test.go index 972689d53..0b4ec698a 100644 --- a/code/go/internal/validator/semantic/validate_fields_limits_test.go +++ b/code/go/internal/validator/semantic/validate_fields_limits_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateFieldsLimits(t *testing.T) { @@ -43,7 +44,7 @@ func TestValidateFieldsLimits(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := validateFieldsLimits(fspath.DirFS(d), 1) + errs := validateFieldsLimits(pkgpath.NewCachedFS(fspath.DirFS(d)), 1) require.Len(t, errs, 1) assert.EqualError(t, errs[0], "data stream foo has more than 1 fields (4)") }) @@ -74,7 +75,7 @@ func TestValidateFieldsLimits(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := validateFieldsLimits(fspath.DirFS(d), 1) + errs := validateFieldsLimits(pkgpath.NewCachedFS(fspath.DirFS(d)), 1) require.Len(t, errs, 1) assert.EqualError(t, errs[0], "transform foo has more than 1 fields (4)") }) @@ -98,7 +99,7 @@ func TestValidateFieldsLimits(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := validateFieldsLimits(fspath.DirFS(d), 1) + errs := validateFieldsLimits(pkgpath.NewCachedFS(fspath.DirFS(d)), 1) require.Len(t, errs, 1) assert.EqualError(t, errs[0], "input package has more than 1 fields (4)") }) diff --git a/code/go/internal/validator/semantic/validate_hbs_templates.go b/code/go/internal/validator/semantic/validate_hbs_templates.go index 5693a6036..bf671f4e0 100644 --- a/code/go/internal/validator/semantic/validate_hbs_templates.go +++ b/code/go/internal/validator/semantic/validate_hbs_templates.go @@ -6,13 +6,11 @@ package semantic import ( "errors" - "io/fs" "os" "path" "github.com/aymerick/raymond" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/linkedfiles" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -24,7 +22,7 @@ var ( // ValidateStaticHandlebarsFiles validates all Handlebars (.hbs) files in the package filesystem. // It returns a list of validation errors if any Handlebars files are invalid. // hbs are located in both the package root and data stream directories under the agent folder. -func ValidateStaticHandlebarsFiles(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateStaticHandlebarsFiles(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors // template files are placed at /agent/input directory or @@ -34,8 +32,8 @@ func ValidateStaticHandlebarsFiles(fsys fspath.FS) specerrors.ValidationErrors { errs = append(errs, inputErrs...) } - datastreamEntries, err := fs.ReadDir(fsys, "data_stream") - if err != nil && !errors.Is(err, fs.ErrNotExist) { + datastreamEntries, err := fsys.Files("data_stream/*") + if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("error reading data_stream directory: %w", err), } @@ -55,9 +53,9 @@ func ValidateStaticHandlebarsFiles(fsys fspath.FS) specerrors.ValidationErrors { } // validateTemplateDir validates all Handlebars files in the given directory. -func validateTemplateDir(fsys fspath.FS, dir string) specerrors.ValidationErrors { - entries, err := fs.ReadDir(fsys, dir) - if err != nil && !errors.Is(err, fs.ErrNotExist) { +func validateTemplateDir(fsys PackageFS, dir string) specerrors.ValidationErrors { + entries, err := fsys.Files(path.Join(dir, "*")) + if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("error trying to read :%s", dir), } @@ -89,7 +87,7 @@ func validateTemplateDir(fsys fspath.FS, dir string) specerrors.ValidationErrors // validateStaticHandlebarsEntry validates a single Handlebars file located at filePath. // it parses the file using the raymond library to check for syntax errors. -func validateStaticHandlebarsEntry(fsys fspath.FS, dir, entryName string) error { +func validateStaticHandlebarsEntry(fsys PackageFS, dir, entryName string) error { if entryName == "" { return nil } @@ -97,14 +95,19 @@ func validateStaticHandlebarsEntry(fsys fspath.FS, dir, entryName string) error var content []byte var err error - // First try to read from filesystem (works for regular files and files within zip) filePath := path.Join(dir, entryName) - if content, err = fs.ReadFile(fsys, filePath); err != nil { - if !errors.Is(err, fs.ErrInvalid) { + files, err := fsys.Files(filePath) + if err != nil { + return err + } + if len(files) > 0 { + content, err = files[0].ReadAll() + if err != nil { return err } - // If fs.ReadFile fails (likely due to linked file path outside filesystem boundary), - // fall back to absolute path approach like linkedfiles.FS does + } else { + // File not found in virtual filesystem, fall back to absolute path + // for linked files pointing outside the filesystem boundary. absolutePath := fsys.Path(filePath) if content, err = os.ReadFile(absolutePath); err != nil { return err diff --git a/code/go/internal/validator/semantic/validate_hbs_templates_test.go b/code/go/internal/validator/semantic/validate_hbs_templates_test.go index a7766215e..8ccd75151 100644 --- a/code/go/internal/validator/semantic/validate_hbs_templates_test.go +++ b/code/go/internal/validator/semantic/validate_hbs_templates_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateTemplateDir(t *testing.T) { @@ -27,7 +28,7 @@ func TestValidateTemplateDir(t *testing.T) { require.NoError(t, err) fsys := fspath.DirFS(pkgDir) - errs := validateTemplateDir(fsys, path.Join("agent", "input")) + errs := validateTemplateDir(pkgpath.NewCachedFS(fsys), path.Join("agent", "input")) require.Empty(t, errs) }) @@ -46,7 +47,7 @@ func TestValidateTemplateDir(t *testing.T) { require.NoError(t, err) fsys := fspath.DirFS(pkgDir) - errs := validateTemplateDir(fsys, path.Join("agent", "input")) + errs := validateTemplateDir(pkgpath.NewCachedFS(fsys), path.Join("agent", "input")) require.Empty(t, errs) }) @@ -65,7 +66,7 @@ func TestValidateTemplateDir(t *testing.T) { require.NoError(t, err) fsys := fspath.DirFS(pkgDir) - errs := validateTemplateDir(fsys, path.Join("agent", "input")) + errs := validateTemplateDir(pkgpath.NewCachedFS(fsys), path.Join("agent", "input")) require.Empty(t, errs) }) t.Run("invalid handlebars file", func(t *testing.T) { @@ -83,7 +84,7 @@ func TestValidateTemplateDir(t *testing.T) { require.NoError(t, err) fsys := fspath.DirFS(pkgDir) - errs := validateTemplateDir(fsys, path.Join("agent", "input")) + errs := validateTemplateDir(pkgpath.NewCachedFS(fsys), path.Join("agent", "input")) require.NotEmpty(t, errs) assert.Len(t, errs, 1) }) @@ -110,7 +111,7 @@ func TestValidateTemplateDir(t *testing.T) { require.NoError(t, err) fsys := fspath.DirFS(pkgDir) - errs := validateTemplateDir(fsys, path.Join("agent", "input")) + errs := validateTemplateDir(pkgpath.NewCachedFS(fsys), path.Join("agent", "input")) require.Empty(t, errs) }) @@ -137,7 +138,7 @@ func TestValidateTemplateDir(t *testing.T) { require.NoError(t, err) fsys := fspath.DirFS(pkgDir) - errs := validateTemplateDir(fsys, path.Join("agent", "input")) + errs := validateTemplateDir(pkgpath.NewCachedFS(fsys), path.Join("agent", "input")) require.NotEmpty(t, errs) assert.Len(t, errs, 1) }) diff --git a/code/go/internal/validator/semantic/validate_ilmpolicypresent.go b/code/go/internal/validator/semantic/validate_ilmpolicypresent.go index 18ae8dd94..3c2bed46b 100644 --- a/code/go/internal/validator/semantic/validate_ilmpolicypresent.go +++ b/code/go/internal/validator/semantic/validate_ilmpolicypresent.go @@ -6,19 +6,17 @@ package semantic import ( "fmt" - "io/fs" "path" "strings" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateILMPolicyPresent produces an error if the indicated ILM policy // is not defined in the data stream. -func ValidateILMPolicyPresent(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateILMPolicyPresent(fsys PackageFS) specerrors.ValidationErrors { dataStreams, err := listDataStreams(fsys) if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} @@ -34,7 +32,7 @@ func ValidateILMPolicyPresent(fsys fspath.FS) specerrors.ValidationErrors { return errs } -func validateILMPolicyInDataStream(fsys fspath.FS, dataStream string) error { +func validateILMPolicyInDataStream(fsys PackageFS, dataStream string) error { dsType, ilmPolicy, err := readILMPolicyInfoInDataStream(fsys, dataStream) if err != nil { return err @@ -57,18 +55,25 @@ func validateILMPolicyInDataStream(fsys fspath.FS, dataStream string) error { ilmFileName := ilmPolicy[len(policyPrefix):] + ".json" ilmFilePath := path.Join("data_stream", dataStream, "elasticsearch", "ilm", ilmFileName) - _, err = fs.Stat(fsys, ilmFilePath) - if err != nil { + ilmFiles, err := fsys.Files(ilmFilePath) + if err != nil || len(ilmFiles) == 0 { return fmt.Errorf("file \"%s\" is invalid: field ilm_policy: ILM policy %q not found in package, expected definition in \"%s\"", fsys.Path(manifestPath), ilmPolicy, fsys.Path(ilmFilePath)) } return nil } -func readILMPolicyInfoInDataStream(fsys fspath.FS, dataStream string) (dsType string, ilmPolicy string, err error) { +func readILMPolicyInfoInDataStream(fsys PackageFS, dataStream string) (dsType string, ilmPolicy string, err error) { manifestPath := path.Join("data_stream", dataStream, "manifest.yml") - d, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return "", "", fmt.Errorf("failed to read data stream manifest in %q: %w", fsys.Path(manifestPath), err) + } + if len(files) == 0 { + return "", "", fmt.Errorf("failed to read data stream manifest in %q: file not found", fsys.Path(manifestPath)) + } + d, err := files[0].ReadAll() if err != nil { return "", "", fmt.Errorf("failed to read data stream manifest in %q: %w", fsys.Path(manifestPath), err) } @@ -85,10 +90,17 @@ func readILMPolicyInfoInDataStream(fsys fspath.FS, dataStream string) (dsType st return manifest.Type, manifest.ILMPolicy, nil } -func readPackageName(fsys fspath.FS) (string, error) { +func readPackageName(fsys PackageFS) (string, error) { manifestPath := "manifest.yml" - d, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return "", fmt.Errorf("failed to manifest in %q: %w", fsys.Path(manifestPath), err) + } + if len(files) == 0 { + return "", fmt.Errorf("failed to manifest in %q: file not found", fsys.Path(manifestPath)) + } + d, err := files[0].ReadAll() if err != nil { return "", fmt.Errorf("failed to manifest in %q: %w", fsys.Path(manifestPath), err) } diff --git a/code/go/internal/validator/semantic/validate_input_dynamic_signal_types.go b/code/go/internal/validator/semantic/validate_input_dynamic_signal_types.go index 0c449b796..a6699623b 100644 --- a/code/go/internal/validator/semantic/validate_input_dynamic_signal_types.go +++ b/code/go/internal/validator/semantic/validate_input_dynamic_signal_types.go @@ -5,11 +5,8 @@ package semantic import ( - "io/fs" - "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -47,12 +44,20 @@ type integrationPackageManifestDynamic struct { } // ValidateInputDynamicSignalTypes validates that dynamic_signal_types field is only used with otelcol input type -func ValidateInputDynamicSignalTypes(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateInputDynamicSignalTypes(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors // Validate package manifest manifestPath := "manifest.yml" - data, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(manifestPath), err)} + } + if len(files) == 0 { + return nil + } + data, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(manifestPath), err)} @@ -83,7 +88,7 @@ func ValidateInputDynamicSignalTypes(fsys fspath.FS) specerrors.ValidationErrors return errs } -func validateInputPackageDynamicSignalTypes(fsys fspath.FS, data []byte, manifestPath string) specerrors.ValidationErrors { +func validateInputPackageDynamicSignalTypes(fsys PackageFS, data []byte, manifestPath string) specerrors.ValidationErrors { var errs specerrors.ValidationErrors var manifest inputPackageManifestDynamic @@ -116,7 +121,7 @@ func validateInputPackageDynamicSignalTypes(fsys fspath.FS, data []byte, manifes return errs } -func validateIntegrationPackageDynamicSignalTypes(fsys fspath.FS, data []byte, manifestPath string) specerrors.ValidationErrors { +func validateIntegrationPackageDynamicSignalTypes(fsys PackageFS, data []byte, manifestPath string) specerrors.ValidationErrors { var errs specerrors.ValidationErrors var manifest integrationPackageManifestDynamic @@ -154,7 +159,7 @@ type dataStreamManifestDynamic struct { Streams []dataStreamStream `yaml:"streams"` } -func validateDataStreamManifests(fsys fspath.FS) specerrors.ValidationErrors { +func validateDataStreamManifests(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors dataStreams, err := listDataStreams(fsys) @@ -165,7 +170,16 @@ func validateDataStreamManifests(fsys fspath.FS) specerrors.ValidationErrors { for _, dataStream := range dataStreams { manifestPath := dataStreamDir + "/" + dataStream + "/manifest.yml" - data, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + errs = append(errs, specerrors.NewStructuredErrorf( + "file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(manifestPath), err)) + continue + } + if len(files) == 0 { + continue + } + data, err := files[0].ReadAll() if err != nil { errs = append(errs, specerrors.NewStructuredErrorf( "file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(manifestPath), err)) diff --git a/code/go/internal/validator/semantic/validate_input_dynamic_signal_types_test.go b/code/go/internal/validator/semantic/validate_input_dynamic_signal_types_test.go index e82a8328e..134487b41 100644 --- a/code/go/internal/validator/semantic/validate_input_dynamic_signal_types_test.go +++ b/code/go/internal/validator/semantic/validate_input_dynamic_signal_types_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateInputDynamicSignalTypes(t *testing.T) { @@ -30,7 +31,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -48,7 +49,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -65,7 +66,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -83,7 +84,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.Contains(t, errs[0].Error(), "dynamic_signal_types is only allowed when input is 'otelcol'") @@ -100,7 +101,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors for non-input packages without field") }) @@ -123,7 +124,7 @@ policy_templates: err = os.Mkdir(d+"/data_stream", 0o755) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors for non-otelcol with dynamic_signal_types") assert.Len(t, errs, 1) assert.Contains(t, errs[0].Error(), "dynamic_signal_types is only allowed when input is 'otelcol'") @@ -141,7 +142,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -163,7 +164,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.Contains(t, errs[0].Error(), "dynamic_signal_types is only allowed when input is 'otelcol'") @@ -186,7 +187,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.Contains(t, errs[0].Error(), "type field must not be set when dynamic_signal_types is true") @@ -206,7 +207,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors when type is present without dynamic_signal_types") }) @@ -224,7 +225,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors when type is present with dynamic_signal_types: false") }) @@ -251,7 +252,7 @@ streams: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors for data stream with otelcol") }) @@ -278,7 +279,7 @@ streams: `), 0o644) require.NoError(t, err) - errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d)) + errs := ValidateInputDynamicSignalTypes(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors for data stream with non-otelcol") assert.Len(t, errs, 1) assert.Contains(t, errs[0].Error(), "dynamic_signal_types is only allowed when input is 'otelcol'") diff --git a/code/go/internal/validator/semantic/validate_input_policy_template_template_path.go b/code/go/internal/validator/semantic/validate_input_policy_template_template_path.go index 13ee5bffa..82bac783b 100644 --- a/code/go/internal/validator/semantic/validate_input_policy_template_template_path.go +++ b/code/go/internal/validator/semantic/validate_input_policy_template_template_path.go @@ -11,7 +11,6 @@ import ( "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -35,11 +34,19 @@ type inputPackageManifest struct { // package manifest } // ValidateInputPackagesPolicyTemplates validates the policy template entries of an input package -func ValidateInputPackagesPolicyTemplates(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateInputPackagesPolicyTemplates(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors manifestPath := "manifest.yml" - data, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %ww", fsys.Path(manifestPath), errFailedToReadManifest)} + } + if len(files) == 0 { + return nil + } + data, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("file \"%s\" is invalid: %ww", fsys.Path(manifestPath), errFailedToReadManifest)} @@ -72,7 +79,7 @@ func ValidateInputPackagesPolicyTemplates(fsys fspath.FS) specerrors.ValidationE // validateInputPackagePolicyTemplate validates the template_path or template_paths at the policy template level for input type packages // if both template_path and template_paths are empty, it returns an error as at least one is required for input type packages -func validateInputPackagePolicyTemplate(fsys fspath.FS, policyTemplate inputPolicyTemplate) error { +func validateInputPackagePolicyTemplate(fsys PackageFS, policyTemplate inputPolicyTemplate) error { if policyTemplate.TemplatePath == "" && len(policyTemplate.TemplatePaths) == 0 { return errRequiredTemplatePath } @@ -94,7 +101,7 @@ func validateInputPackagePolicyTemplate(fsys fspath.FS, policyTemplate inputPoli return nil } -func validateAgentInputTemplatePath(fsys fspath.FS, tmplPath string) error { +func validateAgentInputTemplatePath(fsys PackageFS, tmplPath string) error { dir := path.Join("agent", "input") foundFile, err := findPathAtDirectory(fsys, dir, tmplPath) if err != nil { diff --git a/code/go/internal/validator/semantic/validate_input_policy_template_template_path_test.go b/code/go/internal/validator/semantic/validate_input_policy_template_template_path_test.go index 538ff2bec..347964f21 100644 --- a/code/go/internal/validator/semantic/validate_input_policy_template_template_path_test.go +++ b/code/go/internal/validator/semantic/validate_input_policy_template_template_path_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateInputPackagesPolicyTemplates(t *testing.T) { @@ -32,7 +33,7 @@ policy_templates: err = os.WriteFile(filepath.Join(d, "agent", "input", "udp.yml.hbs"), []byte("# UDP template"), 0o644) require.NoError(t, err) - errs := ValidateInputPackagesPolicyTemplates(fspath.DirFS(d)) + errs := ValidateInputPackagesPolicyTemplates(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -49,7 +50,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputPackagesPolicyTemplates(fspath.DirFS(d)) + errs := ValidateInputPackagesPolicyTemplates(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected no validation errors") assert.Len(t, errs, 1) @@ -69,7 +70,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputPackagesPolicyTemplates(fspath.DirFS(d)) + errs := ValidateInputPackagesPolicyTemplates(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorIs(t, errs[0], errTemplateNotFound) @@ -88,7 +89,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateInputPackagesPolicyTemplates(fspath.DirFS(d)) + errs := ValidateInputPackagesPolicyTemplates(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorIs(t, errs[0], errInvalidPackageType) diff --git a/code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go b/code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go index b393da6a4..7c408c59e 100644 --- a/code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go +++ b/code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go @@ -5,17 +5,14 @@ package semantic import ( - "io/fs" - "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateIntegrationInputsDeprecation checks that if all inputs in an integration package are deprecated, // then the integration package itself must be marked as deprecated. -func ValidateIntegrationInputsDeprecation(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateIntegrationInputsDeprecation(fsys PackageFS) specerrors.ValidationErrors { type manifest struct { Type string `yaml:"type,omitempty"` @@ -34,7 +31,15 @@ func ValidateIntegrationInputsDeprecation(fsys fspath.FS) specerrors.ValidationE } manifestPath := "manifest.yml" - data, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} + } + if len(files) == 0 { + return nil + } + data, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} diff --git a/code/go/internal/validator/semantic/validate_integration_inputs_deprecated_test.go b/code/go/internal/validator/semantic/validate_integration_inputs_deprecated_test.go index 718448be4..abb20bb6e 100644 --- a/code/go/internal/validator/semantic/validate_integration_inputs_deprecated_test.go +++ b/code/go/internal/validator/semantic/validate_integration_inputs_deprecated_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateIntegrationInputsDeprecation(t *testing.T) { @@ -34,7 +35,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d)) + errs := ValidateIntegrationInputsDeprecation(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -53,7 +54,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d)) + errs := ValidateIntegrationInputsDeprecation(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -75,7 +76,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d)) + errs := ValidateIntegrationInputsDeprecation(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -93,7 +94,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d)) + errs := ValidateIntegrationInputsDeprecation(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") require.Len(t, errs, 1) assert.ErrorContains(t, errs[0], "all inputs are deprecated but the integration package is not marked as deprecated") @@ -111,7 +112,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d)) + errs := ValidateIntegrationInputsDeprecation(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -130,7 +131,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidateIntegrationInputsDeprecation(fspath.DirFS(d)) + errs := ValidateIntegrationInputsDeprecation(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) diff --git a/code/go/internal/validator/semantic/validate_integration_policy_template_path.go b/code/go/internal/validator/semantic/validate_integration_policy_template_path.go index 745bb8ac0..ef1f4adef 100644 --- a/code/go/internal/validator/semantic/validate_integration_policy_template_path.go +++ b/code/go/internal/validator/semantic/validate_integration_policy_template_path.go @@ -6,13 +6,11 @@ package semantic import ( "fmt" - "io/fs" "path" "strings" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -46,11 +44,19 @@ type dataStreamManifest struct { } // ValidateIntegrationPolicyTemplates validates the template_path fields at the policy template level for integration type packages -func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateIntegrationPolicyTemplates(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors manifestPath := "manifest.yml" - data, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %ww", fsys.Path(manifestPath), errFailedToReadManifest)} + } + if len(files) == 0 { + return nil + } + data, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("file \"%s\" is invalid: %ww", fsys.Path(manifestPath), errFailedToReadManifest)} @@ -88,7 +94,7 @@ func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErr } // validateIntegrationPackagePolicyTemplate validates the template_path fields at the policy template level for integration type packages -func validateIntegrationPackagePolicyTemplate(fsys fspath.FS, policyTemplate integrationPolicyTemplate, dsManifestMap map[string]dataStreamManifest) error { +func validateIntegrationPackagePolicyTemplate(fsys PackageFS, policyTemplate integrationPolicyTemplate, dsManifestMap map[string]dataStreamManifest) error { for _, input := range policyTemplate.Inputs { if input.TemplatePath != "" { // validate the provided template_path file exists @@ -108,16 +114,16 @@ func validateIntegrationPackagePolicyTemplate(fsys fspath.FS, policyTemplate int } // readDataStreamsManifests reads all data stream manifests and returns a map of data stream directory to its manifest relevant content -func readDataStreamsManifests(fsys fspath.FS) (map[string]dataStreamManifest, error) { +func readDataStreamsManifests(fsys PackageFS) (map[string]dataStreamManifest, error) { // map of data stream directory to its manifest dsManifestMap := make(map[string]dataStreamManifest, 0) - dsManifests, err := fs.Glob(fsys, "data_stream/*/manifest.yml") + dsManifests, err := fsys.Files("data_stream/*/manifest.yml") if err != nil { return nil, err } for _, file := range dsManifests { - data, err := fs.ReadFile(fsys, file) + data, err := file.ReadAll() if err != nil { return nil, err } @@ -127,7 +133,7 @@ func readDataStreamsManifests(fsys fspath.FS) (map[string]dataStreamManifest, er return nil, err } - dsDir := path.Dir(file) + dsDir := path.Dir(file.Path()) dsManifestMap[dsDir] = m } @@ -136,7 +142,7 @@ func readDataStreamsManifests(fsys fspath.FS) (map[string]dataStreamManifest, er // validateInputWithStreams validates that for the given input type, the streams of each dataset related to it have valid template_path files // an input is related to a data_stream if any of its streams has the same input type as input -func validateInputWithStreams(fsys fspath.FS, input string, dsMap map[string]dataStreamManifest) error { +func validateInputWithStreams(fsys PackageFS, input string, dsMap map[string]dataStreamManifest) error { for dsDir, manifest := range dsMap { for _, stream := range manifest.Streams { // only consider streams that match the input type of the policy template @@ -166,9 +172,9 @@ func validateInputWithStreams(fsys fspath.FS, input string, dsMap map[string]dat // findPathAtDirectory looks for a file matching the templatePath in the given directory (dir) // It checks for exact matches, files ending with the templatePath, or templatePath + ".link" -func findPathAtDirectory(fsys fspath.FS, dir, templatePath string) (string, error) { +func findPathAtDirectory(fsys PackageFS, dir, templatePath string) (string, error) { // Check for exact match, files ending with stream.TemplatePath, or stream.TemplatePath + ".link" - entries, err := fs.ReadDir(fsys, dir) + entries, err := fsys.Files(dir + "/*") if err != nil { return "", err } diff --git a/code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go b/code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go index a340ff25c..80d2f65a6 100644 --- a/code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go +++ b/code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestReadDataStreamsManifests(t *testing.T) { @@ -41,7 +42,7 @@ streams: `), 0o644) require.NoError(t, err) - dataStreamsManifestMap, err := readDataStreamsManifests(fspath.DirFS(d)) + dataStreamsManifestMap, err := readDataStreamsManifests(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NoError(t, err) // only the top-level manifest.yml should be read require.Len(t, dataStreamsManifestMap, 1) @@ -83,12 +84,12 @@ func TestValidateInputWithStreams(t *testing.T) { }, } t.Run("valid input with existing template_path", func(t *testing.T) { - err = validateInputWithStreams(fspath.DirFS(d), "nginx/access", dsMap) + err = validateInputWithStreams(pkgpath.NewCachedFS(fspath.DirFS(d)), "nginx/access", dsMap) require.NoError(t, err) }) t.Run("input with non-existing template_path", func(t *testing.T) { - err = validateInputWithStreams(fspath.DirFS(d), "nginx/error", dsMap) + err = validateInputWithStreams(pkgpath.NewCachedFS(fspath.DirFS(d)), "nginx/error", dsMap) require.ErrorIs(t, errTemplateNotFound, err) }) @@ -97,7 +98,7 @@ func TestValidateInputWithStreams(t *testing.T) { require.NoError(t, err) defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", "stream.yml.hbs")) - err = validateInputWithStreams(fspath.DirFS(d), "nginx/other", dsMap) + err = validateInputWithStreams(pkgpath.NewCachedFS(fspath.DirFS(d)), "nginx/other", dsMap) require.NoError(t, err) }) @@ -106,7 +107,7 @@ func TestValidateInputWithStreams(t *testing.T) { require.NoError(t, err) defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", "prefixstream.yml.hbs")) - err = validateInputWithStreams(fspath.DirFS(d), "prefix/stream", dsMap) + err = validateInputWithStreams(pkgpath.NewCachedFS(fspath.DirFS(d)), "prefix/stream", dsMap) require.NoError(t, err) }) @@ -117,7 +118,7 @@ func TestValidateIntegrationPolicyTemplates_NonIntegrationType(t *testing.T) { err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`type: input`), 0o644) require.NoError(t, err) - errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) + errs := ValidateIntegrationPolicyTemplates(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Nil(t, errs) } @@ -147,7 +148,7 @@ streams: err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", "access.yml.hbs"), []byte("template"), 0o644) require.NoError(t, err) - errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) + errs := ValidateIntegrationPolicyTemplates(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs) } @@ -176,7 +177,7 @@ streams: err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", "stream.yml.hbs"), []byte("template"), 0o644) require.NoError(t, err) - errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) + errs := ValidateIntegrationPolicyTemplates(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs) } func TestFindPathAtDirectory(t *testing.T) { @@ -192,7 +193,7 @@ func TestFindPathAtDirectory(t *testing.T) { require.NoError(t, err) defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath)) - foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath) + foundFile, err := findPathAtDirectory(pkgpath.NewCachedFS(fspath.DirFS(d)), dsDir, templatePath) require.NoError(t, err) require.NotEmpty(t, foundFile) require.Equal(t, filepath.ToSlash(path.Join(dsDir, templatePath)), foundFile) @@ -204,7 +205,7 @@ func TestFindPathAtDirectory(t *testing.T) { require.NoError(t, err) defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath+".link")) - foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath) + foundFile, err := findPathAtDirectory(pkgpath.NewCachedFS(fspath.DirFS(d)), dsDir, templatePath) require.NoError(t, err) require.NotEmpty(t, foundFile) require.Equal(t, filepath.ToSlash(path.Join(dsDir, templatePath+".link")), foundFile) @@ -217,7 +218,7 @@ func TestFindPathAtDirectory(t *testing.T) { require.NoError(t, err) defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile)) - foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath) + foundFile, err := findPathAtDirectory(pkgpath.NewCachedFS(fspath.DirFS(d)), dsDir, templatePath) require.NoError(t, err) require.NotEmpty(t, foundFile) require.Equal(t, filepath.ToSlash(path.Join(dsDir, prefixedFile)), foundFile) @@ -230,7 +231,7 @@ func TestFindPathAtDirectory(t *testing.T) { require.NoError(t, err) defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile)) - foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath) + foundFile, err := findPathAtDirectory(pkgpath.NewCachedFS(fspath.DirFS(d)), dsDir, templatePath) require.NoError(t, err) require.NotEmpty(t, foundFile) require.Equal(t, filepath.ToSlash(path.Join(dsDir, prefixedFile)), foundFile) @@ -239,7 +240,7 @@ func TestFindPathAtDirectory(t *testing.T) { t.Run("no match found", func(t *testing.T) { templatePath := "nonexistent.yml.hbs" - foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath) + foundFile, err := findPathAtDirectory(pkgpath.NewCachedFS(fspath.DirFS(d)), dsDir, templatePath) require.NoError(t, err) require.Empty(t, foundFile) }) @@ -257,7 +258,7 @@ func TestFindPathAtDirectory(t *testing.T) { require.NoError(t, err) defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile)) - foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath) + foundFile, err := findPathAtDirectory(pkgpath.NewCachedFS(fspath.DirFS(d)), dsDir, templatePath) require.NoError(t, err) require.NotEmpty(t, foundFile) require.Equal(t, filepath.ToSlash(path.Join(dsDir, exactFile)), foundFile) @@ -276,7 +277,7 @@ func TestFindPathAtDirectory(t *testing.T) { require.NoError(t, err) defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", suffixFile)) - foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath) + foundFile, err := findPathAtDirectory(pkgpath.NewCachedFS(fspath.DirFS(d)), dsDir, templatePath) require.NoError(t, err) require.NotEmpty(t, foundFile) require.Equal(t, filepath.ToSlash(path.Join(dsDir, linkFile)), foundFile) diff --git a/code/go/internal/validator/semantic/validate_kibana_filter_present.go b/code/go/internal/validator/semantic/validate_kibana_filter_present.go index bb248832d..8603e1a80 100644 --- a/code/go/internal/validator/semantic/validate_kibana_filter_present.go +++ b/code/go/internal/validator/semantic/validate_kibana_filter_present.go @@ -13,7 +13,6 @@ import ( "github.com/elastic/kbncontent" "github.com/go-viper/mapstructure/v2" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -26,11 +25,11 @@ var ( // ValidateKibanaFilterPresent checks that all the dashboards included in a package // contain a filter, so only data related to its datasets is queried. -func ValidateKibanaFilterPresent(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateKibanaFilterPresent(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors filePaths := path.Join("kibana", "dashboard", "*.json") - dashboardFiles, err := pkgpath.Files(fsys, filePaths) + dashboardFiles, err := fsys.Files(filePaths) if err != nil { errs = append(errs, specerrors.NewStructuredErrorf("error finding Kibana dashboard files: %w", err)) return errs diff --git a/code/go/internal/validator/semantic/validate_kibana_matching_object_ids.go b/code/go/internal/validator/semantic/validate_kibana_matching_object_ids.go index 5a0d5344a..2939cf6d1 100644 --- a/code/go/internal/validator/semantic/validate_kibana_matching_object_ids.go +++ b/code/go/internal/validator/semantic/validate_kibana_matching_object_ids.go @@ -8,8 +8,6 @@ import ( "path" "strings" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" - "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -17,11 +15,11 @@ import ( // object files that define IDs not matching the file's name. That is, it returns // validation errors if a Kibana object file, foo.json, in the package defines // an object ID other than foo inside it. -func ValidateKibanaObjectIDs(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateKibanaObjectIDs(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors filePaths := path.Join("kibana", "*", "*.json") - objectFiles, err := pkgpath.Files(fsys, filePaths) + objectFiles, err := fsys.Files(filePaths) if err != nil { errs = append(errs, specerrors.NewStructuredErrorf("error finding Kibana object files: %w", err)) return errs diff --git a/code/go/internal/validator/semantic/validate_kibana_no_dangling_object_ids.go b/code/go/internal/validator/semantic/validate_kibana_no_dangling_object_ids.go index 6cb61b03b..b6d969d5a 100644 --- a/code/go/internal/validator/semantic/validate_kibana_no_dangling_object_ids.go +++ b/code/go/internal/validator/semantic/validate_kibana_no_dangling_object_ids.go @@ -9,7 +9,6 @@ import ( "path" "slices" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -29,14 +28,14 @@ var exceptionAssets = []string{ // returns validation errors if a Kibana object file in the package references another // Kibana object with ID i, but no Kibana object file for object ID i is found in the // package. -func ValidateKibanaNoDanglingObjectIDs(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateKibanaNoDanglingObjectIDs(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors installedIDs := []objectReference{} referencedIDs := []objectReference{} filePaths := path.Join("kibana", "*", "*.json") - objectFiles, err := pkgpath.Files(fsys, filePaths) + objectFiles, err := fsys.Files(filePaths) if err != nil { errs = append(errs, specerrors.NewStructuredErrorf("error finding Kibana object files: %w", err)) return errs diff --git a/code/go/internal/validator/semantic/validate_kibana_no_legacy_visualizations.go b/code/go/internal/validator/semantic/validate_kibana_no_legacy_visualizations.go index 9f49ecdd9..a828d93d7 100644 --- a/code/go/internal/validator/semantic/validate_kibana_no_legacy_visualizations.go +++ b/code/go/internal/validator/semantic/validate_kibana_no_legacy_visualizations.go @@ -9,19 +9,17 @@ import ( "github.com/elastic/kbncontent" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" - "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateKibanaNoLegacyVisualizations reports legacy Kibana visualizations in a package. -func ValidateKibanaNoLegacyVisualizations(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateKibanaNoLegacyVisualizations(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors // Collect by-reference visualizations for reference later. // Note: this does not include Lens, Maps, or Discover. That's okay for this rule because none of those are legacy visFilePaths := path.Join("kibana", "visualization", "*.json") - visFiles, _ := pkgpath.Files(fsys, visFilePaths) + visFiles, _ := fsys.Files(visFilePaths) for _, file := range visFiles { filePath := fsys.Path(file.Path()) @@ -54,7 +52,7 @@ func ValidateKibanaNoLegacyVisualizations(fsys fspath.FS) specerrors.ValidationE } dashboardFilePaths := path.Join("kibana", "dashboard", "*.json") - dashboardFiles, err := pkgpath.Files(fsys, dashboardFilePaths) + dashboardFiles, err := fsys.Files(dashboardFilePaths) if err != nil { errs = append(errs, specerrors.NewStructuredErrorf("error finding Kibana dashboard files: %w", err)) return errs diff --git a/code/go/internal/validator/semantic/validate_kibana_tag_duplicates.go b/code/go/internal/validator/semantic/validate_kibana_tag_duplicates.go index 6d1d0b3c4..d053cf649 100644 --- a/code/go/internal/validator/semantic/validate_kibana_tag_duplicates.go +++ b/code/go/internal/validator/semantic/validate_kibana_tag_duplicates.go @@ -6,15 +6,12 @@ package semantic import ( "encoding/json" - "errors" "fmt" - "io/fs" "path" "slices" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -36,7 +33,7 @@ type sharedTagYML struct { // ValidateKibanaTagDuplicates checks for duplicate Kibana tag names // between the kibana/tags.yml file and the tags defined in the package's kibana/tag/*.json files. // It returns a list of validation errors if any duplicates are found. -func ValidateKibanaTagDuplicates(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateKibanaTagDuplicates(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors sharedTagNames, verr := getValidatedSharedKibanaTags(fsys) if len(verr) > 0 { @@ -52,15 +49,18 @@ func ValidateKibanaTagDuplicates(fsys fspath.FS) specerrors.ValidationErrors { // getValidatedSharedKibanaTags reads the kibana/tags.yml file and returns a slice of tag names defined in it. // It also returns any validation errors encountered during the process if tags are duplicated within the file. -func getValidatedSharedKibanaTags(fsys fspath.FS) ([]string, specerrors.ValidationErrors) { +func getValidatedSharedKibanaTags(fsys PackageFS) ([]string, specerrors.ValidationErrors) { tagsPath := path.Join("kibana", "tags.yml") // Collect all tags defined in the kibana/tags.yml file. - b, err := fs.ReadFile(fsys, tagsPath) + files, err := fsys.Files(tagsPath) + if err != nil { + return nil, specerrors.ValidationErrors{specerrors.NewStructuredErrorf("error reading file %s: %v", tagsPath, err)} + } + if len(files) == 0 { + return nil, nil + } + b, err := files[0].ReadAll() if err != nil { - // if the file does not exist, return an empty slice without error - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } return nil, specerrors.ValidationErrors{specerrors.NewStructuredErrorf("error reading file %s: %v", tagsPath, err)} } var sharedKibanaTags []sharedTagYML @@ -83,14 +83,14 @@ func getValidatedSharedKibanaTags(fsys fspath.FS) ([]string, specerrors.Validati return tags, errs } -func validateKibanaPackageTagsDuplicates(fsys fspath.FS, sharedTagNames []string) specerrors.ValidationErrors { - entries, err := fs.ReadDir(fsys, path.Join("kibana", "tag")) +func validateKibanaPackageTagsDuplicates(fsys PackageFS, sharedTagNames []string) specerrors.ValidationErrors { + entries, err := fsys.Files(path.Join("kibana", "tag", "*")) if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil - } return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("error reading kibana/tag directory: %v", err)} } + if len(entries) == 0 { + return nil + } tags := make([]string, 0) errs := make(specerrors.ValidationErrors, 0) @@ -99,16 +99,15 @@ func validateKibanaPackageTagsDuplicates(fsys fspath.FS, sharedTagNames []string // skip non-json files and directories continue } - filePath := path.Join("kibana", "tag", entry.Name()) - b, err := fs.ReadFile(fsys, filePath) + b, err := entry.ReadAll() if err != nil { - errs = append(errs, specerrors.NewStructuredErrorf("error reading file %s: %v", fsys.Path(filePath), err)) + errs = append(errs, specerrors.NewStructuredErrorf("error reading file %s: %v", fsys.Path(entry.Path()), err)) continue } var pkgTag packageSpecTag err = json.Unmarshal(b, &pkgTag) if err != nil { - errs = append(errs, specerrors.NewStructuredErrorf("error unmarshaling file %s: %v", fsys.Path(filePath), err)) + errs = append(errs, specerrors.NewStructuredErrorf("error unmarshaling file %s: %v", fsys.Path(entry.Path()), err)) continue } // skip non-tag types @@ -119,12 +118,12 @@ func validateKibanaPackageTagsDuplicates(fsys fspath.FS, sharedTagNames []string // validate if the tag is already defined in other json file if slices.Contains(tags, pkgTag.Attributes.Name) { errs = append(errs, specerrors.NewStructuredError( - fmt.Errorf("file \"%s\" is invalid: duplicate package tag name '%s' found", fsys.Path(filePath), pkgTag.Attributes.Name), specerrors.CodeKibanaTagDuplicates)) + fmt.Errorf("file \"%s\" is invalid: duplicate package tag name '%s' found", fsys.Path(entry.Path()), pkgTag.Attributes.Name), specerrors.CodeKibanaTagDuplicates)) continue } if slices.Contains(sharedTagNames, pkgTag.Attributes.Name) { errs = append(errs, specerrors.NewStructuredError( - fmt.Errorf("file \"%s\" is invalid: tag name '%s' is already defined in tags.yml", fsys.Path(filePath), pkgTag.Attributes.Name), specerrors.CodeKibanaTagDuplicates)) + fmt.Errorf("file \"%s\" is invalid: tag name '%s' is already defined in tags.yml", fsys.Path(entry.Path()), pkgTag.Attributes.Name), specerrors.CodeKibanaTagDuplicates)) continue } tags = append(tags, pkgTag.Attributes.Name) diff --git a/code/go/internal/validator/semantic/validate_kibana_tag_duplicates_test.go b/code/go/internal/validator/semantic/validate_kibana_tag_duplicates_test.go index ad8433f85..b99089c09 100644 --- a/code/go/internal/validator/semantic/validate_kibana_tag_duplicates_test.go +++ b/code/go/internal/validator/semantic/validate_kibana_tag_duplicates_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestGetValidatedSharedKibanaTags(t *testing.T) { @@ -20,7 +21,7 @@ func TestGetValidatedSharedKibanaTags(t *testing.T) { tmpDir := t.TempDir() fsys := fspath.DirFS(tmpDir) - tags, errs := getValidatedSharedKibanaTags(fsys) + tags, errs := getValidatedSharedKibanaTags(pkgpath.NewCachedFS(fsys)) require.Empty(t, errs) assert.Empty(t, tags) }) @@ -40,7 +41,7 @@ func TestGetValidatedSharedKibanaTags(t *testing.T) { require.NoError(t, err) fsys := fspath.DirFS(tmpDir) - tags, errs := getValidatedSharedKibanaTags(fsys) + tags, errs := getValidatedSharedKibanaTags(pkgpath.NewCachedFS(fsys)) require.Len(t, errs, 1) assert.Contains(t, errs[0].Error(), "duplicate tag name 'tag1' found (SVR00007)") require.Len(t, tags, 2) @@ -78,7 +79,7 @@ func TestValidateKibanaPackageTagsDuplicates(t *testing.T) { fsys := fspath.DirFS(tmpDir) tags := []string{"tagB"} - errs := validateKibanaPackageTagsDuplicates(fsys, tags) + errs := validateKibanaPackageTagsDuplicates(pkgpath.NewCachedFS(fsys), tags) require.Len(t, errs, 1) assert.Contains(t, errs[0].Error(), "duplicate package tag name 'tagA'") }) @@ -101,7 +102,7 @@ func TestValidateKibanaPackageTagsDuplicates(t *testing.T) { fsys := fspath.DirFS(tmpDir) tags := []string{"tagB"} - errs := validateKibanaPackageTagsDuplicates(fsys, tags) + errs := validateKibanaPackageTagsDuplicates(pkgpath.NewCachedFS(fsys), tags) require.Len(t, errs, 1) assert.Contains(t, errs[0].Error(), "tag name 'tagB' is already defined in tags.yml (SVR00007)") }) @@ -134,7 +135,7 @@ func TestValidateKibanaPackageTagsDuplicates(t *testing.T) { fsys := fspath.DirFS(tmpDir) tags := []string{"tagC"} - errs := validateKibanaPackageTagsDuplicates(fsys, tags) + errs := validateKibanaPackageTagsDuplicates(pkgpath.NewCachedFS(fsys), tags) require.Empty(t, errs) }) } diff --git a/code/go/internal/validator/semantic/validate_minimum_kibana_version.go b/code/go/internal/validator/semantic/validate_minimum_kibana_version.go index 8515be576..6c9de1f4a 100644 --- a/code/go/internal/validator/semantic/validate_minimum_kibana_version.go +++ b/code/go/internal/validator/semantic/validate_minimum_kibana_version.go @@ -10,15 +10,18 @@ import ( "github.com/Masterminds/semver/v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/packages" "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateMinimumKibanaVersion ensures the minimum kibana version for a given package is the expected one -func ValidateMinimumKibanaVersion(fsys fspath.FS) specerrors.ValidationErrors { - pkg, err := packages.NewPackageFromFS(fsys.Path(), fsys) +func ValidateMinimumKibanaVersion(fsys PackageFS) specerrors.ValidationErrors { + cached, ok := fsys.(*pkgpath.CachedFS) + if !ok { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("unexpected filesystem type for minimum kibana version validation")} + } + pkg, err := packages.NewPackageFromFS(fsys.Path(), cached.RawFS()) if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} } @@ -77,7 +80,7 @@ func validateMinimumKibanaVersionInputPackages(packageType string, packageVersio // validateMinimumKibanaVersionRuntimeFields ensures the minimum kibana version if the package defines any runtime field, // then the kibana version condition for the package must be >= 8.10.0 -func validateMinimumKibanaVersionRuntimeFields(fsys fspath.FS, packageVersion semver.Version, kibanaVersionCondition string) error { +func validateMinimumKibanaVersionRuntimeFields(fsys PackageFS, packageVersion semver.Version, kibanaVersionCondition string) error { const minimumKibanaVersion = "8.10.0" errs := validateFields(fsys, validateNoRuntimeFields) if len(errs) == 0 { @@ -93,14 +96,14 @@ func validateMinimumKibanaVersionRuntimeFields(fsys fspath.FS, packageVersion se // validateMinimumKibanaVersionSavedObjectTags ensures the minimum kibana version if the package defines saved object tags file, // then the kibana version condition for the package must be >= 8.10.0 -func validateMinimumKibanaVersionSavedObjectTags(fsys fspath.FS, packageType string, packageVersion semver.Version, kibanaVersionCondition string) error { +func validateMinimumKibanaVersionSavedObjectTags(fsys PackageFS, packageType string, packageVersion semver.Version, kibanaVersionCondition string) error { const minimumKibanaVersion = "8.10.0" if packageType == "input" { return nil } manifestPath := "kibana/tags.yml" - f, err := pkgpath.Files(fsys, manifestPath) + f, err := fsys.Files(manifestPath) if err != nil { return fmt.Errorf("can't locate files with %v: %w", manifestPath, err) } @@ -116,9 +119,9 @@ func validateMinimumKibanaVersionSavedObjectTags(fsys fspath.FS, packageType str return fmt.Errorf("conditions.kibana.version must be ^%s or greater to include saved object tags file: %s", minimumKibanaVersion, manifestPath) } -func readManifest(fsys fspath.FS) (*pkgpath.File, error) { +func readManifest(fsys PackageFS) (*pkgpath.File, error) { manifestPath := "manifest.yml" - f, err := pkgpath.Files(fsys, manifestPath) + f, err := fsys.Files(manifestPath) if err != nil { return nil, fmt.Errorf("can't locate manifest file: %w", err) } diff --git a/code/go/internal/validator/semantic/validate_minimum_kibana_version_test.go b/code/go/internal/validator/semantic/validate_minimum_kibana_version_test.go index 23b2ff850..c5276927e 100644 --- a/code/go/internal/validator/semantic/validate_minimum_kibana_version_test.go +++ b/code/go/internal/validator/semantic/validate_minimum_kibana_version_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateKibanaVersionGreaterThan(t *testing.T) { @@ -161,7 +162,7 @@ func TestValidateMinimumKibanaVersionRuntimeFields(t *testing.T) { for _, test := range tests { t.Run(filepath.Base(test.pkgRoot)+"--"+test.packageVersion.String()+"--"+test.kibanaVersionCondition, func(t *testing.T) { - res := validateMinimumKibanaVersionRuntimeFields(fspath.DirFS(test.pkgRoot), test.packageVersion, test.kibanaVersionCondition) + res := validateMinimumKibanaVersionRuntimeFields(pkgpath.NewCachedFS(fspath.DirFS(test.pkgRoot)), test.packageVersion, test.kibanaVersionCondition) if test.expectedErr == nil { assert.Nil(t, res) @@ -214,7 +215,7 @@ func TestValidateMinimumKibanaVersionSavedObjectsTags(t *testing.T) { for _, test := range tests { t.Run(filepath.Base(test.pkgRoot)+"--"+test.packageVersion.String()+"--"+test.kibanaVersionCondition, func(t *testing.T) { - res := validateMinimumKibanaVersionSavedObjectTags(fspath.DirFS(test.pkgRoot), test.pkgType, test.packageVersion, test.kibanaVersionCondition) + res := validateMinimumKibanaVersionSavedObjectTags(pkgpath.NewCachedFS(fspath.DirFS(test.pkgRoot)), test.pkgType, test.packageVersion, test.kibanaVersionCondition) if test.expectedErr == nil { assert.Nil(t, res) diff --git a/code/go/internal/validator/semantic/validate_package_references.go b/code/go/internal/validator/semantic/validate_package_references.go index 6c24c54f8..ecb085204 100644 --- a/code/go/internal/validator/semantic/validate_package_references.go +++ b/code/go/internal/validator/semantic/validate_package_references.go @@ -9,14 +9,13 @@ import ( "github.com/Masterminds/semver/v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidatePackageReferences checks that package references in policy templates and data streams // are listed in the manifest's requires section and are of the correct type (input packages only). -func ValidatePackageReferences(fsys fspath.FS) specerrors.ValidationErrors { +func ValidatePackageReferences(fsys PackageFS) specerrors.ValidationErrors { manifest, err := readManifest(fsys) if err != nil { return specerrors.ValidationErrors{ @@ -97,7 +96,7 @@ func extractPackageNamesFromRequires(packages interface{}) []requiredPackage { return result } -func validatePolicyTemplatePackageReferences(fsys fspath.FS, manifest pkgpath.File, requiredPackages requiredPackages) specerrors.ValidationErrors { +func validatePolicyTemplatePackageReferences(fsys PackageFS, manifest pkgpath.File, requiredPackages requiredPackages) specerrors.ValidationErrors { var errs specerrors.ValidationErrors policyTemplates, err := manifest.Values("$.policy_templates") @@ -156,8 +155,8 @@ func validatePolicyTemplatePackageReferences(fsys fspath.FS, manifest pkgpath.Fi return errs } -func validateDataStreamPackageReferences(fsys fspath.FS, requiredPackages requiredPackages) specerrors.ValidationErrors { - dataStreamManifests, err := pkgpath.Files(fsys, "data_stream/*/manifest.yml") +func validateDataStreamPackageReferences(fsys PackageFS, requiredPackages requiredPackages) specerrors.ValidationErrors { + dataStreamManifests, err := fsys.Files("data_stream/*/manifest.yml") if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("error while searching for data stream manifests: %w", err)} @@ -210,7 +209,7 @@ func validateDataStreamPackageReferences(fsys fspath.FS, requiredPackages requir return errs } -func validateFixedVersions(fsys fspath.FS, requiredPackages requiredPackages) specerrors.ValidationErrors { +func validateFixedVersions(fsys PackageFS, requiredPackages requiredPackages) specerrors.ValidationErrors { var errs specerrors.ValidationErrors // Validate that input packages have fixed versions, and no constraints. diff --git a/code/go/internal/validator/semantic/validate_package_references_test.go b/code/go/internal/validator/semantic/validate_package_references_test.go index 01760768e..df7c29246 100644 --- a/code/go/internal/validator/semantic/validate_package_references_test.go +++ b/code/go/internal/validator/semantic/validate_package_references_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidatePackageReferences(t *testing.T) { @@ -35,7 +36,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidatePackageReferences(fspath.DirFS(d)) + errs := ValidatePackageReferences(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -58,7 +59,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidatePackageReferences(fspath.DirFS(d)) + errs := ValidatePackageReferences(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, `policy_templates[0].inputs[0] references package "missing_package" which is not listed in requires section`) @@ -83,7 +84,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidatePackageReferences(fspath.DirFS(d)) + errs := ValidatePackageReferences(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, `policy_templates[0].inputs[0] references package "apache_otel" which is a content package, only input packages allowed`) @@ -115,7 +116,7 @@ streams: `), 0o644) require.NoError(t, err) - errs := ValidatePackageReferences(fspath.DirFS(d)) + errs := ValidatePackageReferences(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Empty(t, errs, "expected no validation errors") }) @@ -145,7 +146,7 @@ streams: `), 0o644) require.NoError(t, err) - errs := ValidatePackageReferences(fspath.DirFS(d)) + errs := ValidatePackageReferences(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, `streams[0] references package "missing_package" which is not listed in manifest requires section`) @@ -177,7 +178,7 @@ streams: `), 0o644) require.NoError(t, err) - errs := ValidatePackageReferences(fspath.DirFS(d)) + errs := ValidatePackageReferences(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, `streams[0] references package "security_rules" which is a content package, only input packages allowed`) @@ -198,7 +199,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidatePackageReferences(fspath.DirFS(d)) + errs := ValidatePackageReferences(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 1) assert.ErrorContains(t, errs, `policy_templates[0].inputs[0] references package "some_package" which is not listed in requires section`) @@ -226,7 +227,7 @@ policy_templates: `), 0o644) require.NoError(t, err) - errs := ValidatePackageReferences(fspath.DirFS(d)) + errs := ValidatePackageReferences(pkgpath.NewCachedFS(fspath.DirFS(d))) require.NotEmpty(t, errs, "expected validation errors") assert.Len(t, errs, 2) assert.ErrorContains(t, errs, `policy_templates[0].inputs[0] references package "missing_package_1"`) diff --git a/code/go/internal/validator/semantic/validate_pipeline_on_failure.go b/code/go/internal/validator/semantic/validate_pipeline_on_failure.go index 4654a8613..69c10f163 100644 --- a/code/go/internal/validator/semantic/validate_pipeline_on_failure.go +++ b/code/go/internal/validator/semantic/validate_pipeline_on_failure.go @@ -6,12 +6,10 @@ package semantic import ( "fmt" - "io/fs" "strings" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -23,7 +21,7 @@ var requiredMessageValues = []string{ } // ValidatePipelineOnFailure validates ingest pipeline global on_failure handlers. -func ValidatePipelineOnFailure(fsys fspath.FS) specerrors.ValidationErrors { +func ValidatePipelineOnFailure(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors pipelineFiles, err := listPipelineFiles(fsys) @@ -32,7 +30,14 @@ func ValidatePipelineOnFailure(fsys fspath.FS) specerrors.ValidationErrors { } for _, pipelineFile := range pipelineFiles { - content, err := fs.ReadFile(fsys, pipelineFile.filePath) + files, err := fsys.Files(pipelineFile.filePath) + if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} + } + if len(files) == 0 { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("pipeline file not found: %s", pipelineFile.filePath)} + } + content, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} } diff --git a/code/go/internal/validator/semantic/validate_pipeline_tags.go b/code/go/internal/validator/semantic/validate_pipeline_tags.go index bd4e8246d..4c4b304ca 100644 --- a/code/go/internal/validator/semantic/validate_pipeline_tags.go +++ b/code/go/internal/validator/semantic/validate_pipeline_tags.go @@ -6,16 +6,14 @@ package semantic import ( "fmt" - "io/fs" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidatePipelineTags validates ingest pipeline processor tags. -func ValidatePipelineTags(fsys fspath.FS) specerrors.ValidationErrors { +func ValidatePipelineTags(fsys PackageFS) specerrors.ValidationErrors { var errors specerrors.ValidationErrors pipelineFiles, err := listPipelineFiles(fsys) if err != nil { @@ -23,7 +21,14 @@ func ValidatePipelineTags(fsys fspath.FS) specerrors.ValidationErrors { } for _, pipelineFile := range pipelineFiles { - content, err := fs.ReadFile(fsys, pipelineFile.filePath) + files, err := fsys.Files(pipelineFile.filePath) + if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} + } + if len(files) == 0 { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("pipeline file not found: %s", pipelineFile.filePath)} + } + content, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} } diff --git a/code/go/internal/validator/semantic/validate_prerelease.go b/code/go/internal/validator/semantic/validate_prerelease.go index f6e5ee08f..8bf6aaab6 100644 --- a/code/go/internal/validator/semantic/validate_prerelease.go +++ b/code/go/internal/validator/semantic/validate_prerelease.go @@ -11,7 +11,6 @@ import ( "github.com/Masterminds/semver/v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -32,7 +31,7 @@ var ( ) // ValidatePrerelease validates additional restrictions on the prerelease tags. -func ValidatePrerelease(fsys fspath.FS) specerrors.ValidationErrors { +func ValidatePrerelease(fsys PackageFS) specerrors.ValidationErrors { manifestVersion, err := readManifestVersion(fsys) if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} diff --git a/code/go/internal/validator/semantic/validate_profiling_nonga.go b/code/go/internal/validator/semantic/validate_profiling_nonga.go index 8bb9f526a..8be75941d 100644 --- a/code/go/internal/validator/semantic/validate_profiling_nonga.go +++ b/code/go/internal/validator/semantic/validate_profiling_nonga.go @@ -6,19 +6,17 @@ package semantic import ( "fmt" - "io/fs" "path" "github.com/Masterminds/semver/v3" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateProfilingNonGA validates that the profiling data type is not used in GA packages, // as this data type is in technical preview and can be eventually removed. -func ValidateProfilingNonGA(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateProfilingNonGA(fsys PackageFS) specerrors.ValidationErrors { manifestVersion, err := readManifestVersion(fsys) if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} @@ -48,9 +46,16 @@ func ValidateProfilingNonGA(fsys fspath.FS) specerrors.ValidationErrors { return errs } -func validateProfilingTypeNotUsed(fsys fspath.FS, dataStream string) error { +func validateProfilingTypeNotUsed(fsys PackageFS, dataStream string) error { manifestPath := path.Join("data_stream", dataStream, "manifest.yml") - d, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return fmt.Errorf("failed to read data stream manifest in \"%s\": %w", fsys.Path(manifestPath), err) + } + if len(files) == 0 { + return fmt.Errorf("failed to read data stream manifest in \"%s\": file not found", fsys.Path(manifestPath)) + } + d, err := files[0].ReadAll() if err != nil { return fmt.Errorf("failed to read data stream manifest in \"%s\": %w", fsys.Path(manifestPath), err) } diff --git a/code/go/internal/validator/semantic/validate_required_fields.go b/code/go/internal/validator/semantic/validate_required_fields.go index 20f8de9a4..792dec0c8 100644 --- a/code/go/internal/validator/semantic/validate_required_fields.go +++ b/code/go/internal/validator/semantic/validate_required_fields.go @@ -7,13 +7,12 @@ package semantic import ( "fmt" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateRequiredFields validates that required fields are present and have the expected // types except for fields defined in transforms. -func ValidateRequiredFields(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateRequiredFields(fsys PackageFS) specerrors.ValidationErrors { requiredFields := map[string]string{ "data_stream.type": "constant_keyword", "data_stream.dataset": "constant_keyword", @@ -53,7 +52,7 @@ func (e notFoundRequiredField) Error() string { return message } -func validateRequiredFields(fsys fspath.FS, requiredFields map[string]string) specerrors.ValidationErrors { +func validateRequiredFields(fsys PackageFS, requiredFields map[string]string) specerrors.ValidationErrors { // map datastream/input package -> field name -> found // if data stream is an empty string, it means it is an input package foundFields := make(map[string]map[string]struct{}) diff --git a/code/go/internal/validator/semantic/validate_required_fields_test.go b/code/go/internal/validator/semantic/validate_required_fields_test.go index 44b99298b..1acee46ad 100644 --- a/code/go/internal/validator/semantic/validate_required_fields_test.go +++ b/code/go/internal/validator/semantic/validate_required_fields_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateRequiredFields(t *testing.T) { @@ -32,7 +33,7 @@ func TestValidateRequiredFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateRequiredFields(fspath.DirFS(d)) + errs := ValidateRequiredFields(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Len(t, errs, 0) }) t.Run("missing required fields", func(t *testing.T) { @@ -46,7 +47,7 @@ func TestValidateRequiredFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateRequiredFields(fspath.DirFS(d)) + errs := ValidateRequiredFields(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Len(t, errs, 3) }) t.Run("required fields with incorrect types", func(t *testing.T) { @@ -66,7 +67,7 @@ func TestValidateRequiredFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateRequiredFields(fspath.DirFS(d)) + errs := ValidateRequiredFields(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Len(t, errs, 4) }) @@ -87,7 +88,7 @@ func TestValidateRequiredFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateRequiredFields(fspath.DirFS(d)) + errs := ValidateRequiredFields(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Len(t, errs, 0) }) t.Run("missing required fields in transform", func(t *testing.T) { @@ -101,7 +102,7 @@ func TestValidateRequiredFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateRequiredFields(fspath.DirFS(d)) + errs := ValidateRequiredFields(pkgpath.NewCachedFS(fspath.DirFS(d))) // Ignored missing required fields in transform require.Len(t, errs, 0) }) @@ -122,7 +123,7 @@ func TestValidateRequiredFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateRequiredFields(fspath.DirFS(d)) + errs := ValidateRequiredFields(pkgpath.NewCachedFS(fspath.DirFS(d))) // should data_stream.type, data_stream.dataset, data_stream.namespace fields be enforced as constant_keyword too? // should @timestamp be enforced as date too? // Ignored incorrect types for required fields in transform @@ -140,7 +141,7 @@ func TestValidateRequiredFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateRequiredFields(fspath.DirFS(d)) + errs := ValidateRequiredFields(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Len(t, errs, 3) }) } diff --git a/code/go/internal/validator/semantic/validate_required_vargroups.go b/code/go/internal/validator/semantic/validate_required_vargroups.go index 064f354dd..33ad5d0cc 100644 --- a/code/go/internal/validator/semantic/validate_required_vargroups.go +++ b/code/go/internal/validator/semantic/validate_required_vargroups.go @@ -5,20 +5,25 @@ package semantic import ( - "io/fs" "path" "slices" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateRequiredVarGroups validates lists of optional required variables. -func ValidateRequiredVarGroups(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateRequiredVarGroups(fsys PackageFS) specerrors.ValidationErrors { // Validate main manifest. - d, err := fs.ReadFile(fsys, "manifest.yml") + files, err := fsys.Files("manifest.yml") + if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path("manifest.yml"), err)} + } + if len(files) == 0 { + return nil + } + d, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path("manifest.yml"), err)} } @@ -87,19 +92,26 @@ func validateRequiredVarGroupsManifest(path string, manifest requiredVarsManifes return errs } -func validateDataStreamRequiredVarGroups(fsys fspath.FS, path string, pkgManifest requiredVarsManifest) specerrors.ValidationErrors { - d, err := fs.ReadFile(fsys, path) +func validateDataStreamRequiredVarGroups(fsys PackageFS, filePath string, pkgManifest requiredVarsManifest) specerrors.ValidationErrors { + files, err := fsys.Files(filePath) + if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(filePath), err)} + } + if len(files) == 0 { + return nil + } + d, err := files[0].ReadAll() if err != nil { - return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(path), err)} + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(filePath), err)} } var manifest requiredVarsDataStreamManifest err = yaml.Unmarshal(d, &manifest) if err != nil { - return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to parse manifest: %w", fsys.Path(path), err)} + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to parse manifest: %w", fsys.Path(filePath), err)} } - return validateDataStreamRequiredVarGroupsManifest(fsys.Path(path), manifest, pkgManifest) + return validateDataStreamRequiredVarGroupsManifest(fsys.Path(filePath), manifest, pkgManifest) } type requiredVarsDataStreamManifest struct { diff --git a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go index 5a69f89c0..08e0578bc 100644 --- a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go +++ b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go @@ -6,19 +6,16 @@ package semantic import ( "fmt" - "io/fs" "path" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" - "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateRoutingRulesAndDataset returns validation errors if there are routing rules defined in any dataStream // but that dataStream does not defines "dataset" field. -func ValidateRoutingRulesAndDataset(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateRoutingRulesAndDataset(fsys PackageFS) specerrors.ValidationErrors { dataStreams, err := listDataStreams(fsys) if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} @@ -46,9 +43,16 @@ func ValidateRoutingRulesAndDataset(fsys fspath.FS) specerrors.ValidationErrors return errs } -func validateDatasetInDataStream(fsys fspath.FS, dataStream string) error { +func validateDatasetInDataStream(fsys PackageFS, dataStream string) error { manifestPath := path.Join("data_stream", dataStream, "manifest.yml") - d, err := fs.ReadFile(fsys, manifestPath) + files, err := fsys.Files(manifestPath) + if err != nil { + return fmt.Errorf("failed to read data stream manifest in %q: %w", fsys.Path(manifestPath), err) + } + if len(files) == 0 { + return fmt.Errorf("failed to read data stream manifest in %q: file not found", fsys.Path(manifestPath)) + } + d, err := files[0].ReadAll() if err != nil { return fmt.Errorf("failed to read data stream manifest in %q: %w", fsys.Path(manifestPath), err) } @@ -67,9 +71,9 @@ func validateDatasetInDataStream(fsys fspath.FS, dataStream string) error { return nil } -func anyRoutingRulesInDataStream(fsys fspath.FS, dataStream string) (bool, error) { +func anyRoutingRulesInDataStream(fsys PackageFS, dataStream string) (bool, error) { routingRulesPath := path.Join("data_stream", dataStream, "routing_rules.yml") - f, err := pkgpath.Files(fsys, routingRulesPath) + f, err := fsys.Files(routingRulesPath) if err != nil { return false, nil } diff --git a/code/go/internal/validator/semantic/validate_test_package_requirements.go b/code/go/internal/validator/semantic/validate_test_package_requirements.go index a5f5bbe32..f0c52a570 100644 --- a/code/go/internal/validator/semantic/validate_test_package_requirements.go +++ b/code/go/internal/validator/semantic/validate_test_package_requirements.go @@ -11,14 +11,13 @@ import ( "github.com/Masterminds/semver/v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateTestPackageRequirements checks that package requirements in test configurations // reference packages listed in the manifest and that versions satisfy constraints. -func ValidateTestPackageRequirements(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateTestPackageRequirements(fsys PackageFS) specerrors.ValidationErrors { manifest, err := readManifest(fsys) if err != nil { return specerrors.ValidationErrors{ @@ -95,8 +94,8 @@ func getRequiredPackagesWithConstraints(manifest pkgpath.File) (map[string]strin return requiredPackages, nil } -func validateIntegrationTestRequirements(fsys fspath.FS, requiredPackages map[string]string) specerrors.ValidationErrors { - testConfig, err := pkgpath.Files(fsys, "_dev/test/config.yml") +func validateIntegrationTestRequirements(fsys PackageFS, requiredPackages map[string]string) specerrors.ValidationErrors { + testConfig, err := fsys.Files("_dev/test/config.yml") if err != nil || len(testConfig) == 0 { return nil } @@ -141,7 +140,7 @@ func validateIntegrationTestRequirements(fsys fspath.FS, requiredPackages map[st return errs } -func validateDataStreamTestRequirements(fsys fspath.FS, requiredPackages map[string]string) specerrors.ValidationErrors { +func validateDataStreamTestRequirements(fsys PackageFS, requiredPackages map[string]string) specerrors.ValidationErrors { var errs specerrors.ValidationErrors patterns := []string{ @@ -151,7 +150,7 @@ func validateDataStreamTestRequirements(fsys fspath.FS, requiredPackages map[str } for _, pattern := range patterns { - testConfigs, err := pkgpath.Files(fsys, pattern) + testConfigs, err := fsys.Files(pattern) if err != nil { continue } diff --git a/code/go/internal/validator/semantic/validate_test_package_requirements_test.go b/code/go/internal/validator/semantic/validate_test_package_requirements_test.go index ea10927f9..e3870df19 100644 --- a/code/go/internal/validator/semantic/validate_test_package_requirements_test.go +++ b/code/go/internal/validator/semantic/validate_test_package_requirements_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateTestPackageRequirements(t *testing.T) { @@ -206,7 +207,7 @@ format_version: 3.6.0`, require.NoError(t, err) fsys := fspath.DirFS(pkgRoot) - errs := ValidateTestPackageRequirements(fsys) + errs := ValidateTestPackageRequirements(pkgpath.NewCachedFS(fsys)) if tc.expectError { require.NotEmpty(t, errs, "expected validation errors but got none") diff --git a/code/go/internal/validator/semantic/validate_unique_fields.go b/code/go/internal/validator/semantic/validate_unique_fields.go index ff1a7e3f5..d5d59ab4b 100644 --- a/code/go/internal/validator/semantic/validate_unique_fields.go +++ b/code/go/internal/validator/semantic/validate_unique_fields.go @@ -9,7 +9,6 @@ import ( "sort" "strings" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -20,7 +19,7 @@ type uniqueField struct { } // ValidateUniqueFields verifies that any field is defined only once on each data stream. -func ValidateUniqueFields(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateUniqueFields(fsys PackageFS) specerrors.ValidationErrors { // data_stream -> field -> files // if data stream is empty string, it means it is an input package fields := make(map[string]map[uniqueField][]string) diff --git a/code/go/internal/validator/semantic/validate_unique_fields_test.go b/code/go/internal/validator/semantic/validate_unique_fields_test.go index 63da949ab..6d02f37b0 100644 --- a/code/go/internal/validator/semantic/validate_unique_fields_test.go +++ b/code/go/internal/validator/semantic/validate_unique_fields_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" ) func TestValidateUniqueFields(t *testing.T) { @@ -43,7 +44,7 @@ func TestValidateUniqueFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateUniqueFields(fspath.DirFS(d)) + errs := ValidateUniqueFields(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Len(t, errs, 0) }) t.Run("non-unique fields across data streams", func(t *testing.T) { @@ -78,7 +79,7 @@ func TestValidateUniqueFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateUniqueFields(fspath.DirFS(d)) + errs := ValidateUniqueFields(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Len(t, errs, 1) assert.ErrorContains(t, errs[0], "field \"field2\" is defined multiple times for data stream \"foo\", found in:") }) @@ -111,7 +112,7 @@ func TestValidateUniqueFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateUniqueFields(fspath.DirFS(d)) + errs := ValidateUniqueFields(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Len(t, errs, 1) assert.ErrorContains(t, errs[0], "field \"field2\" is defined multiple times for transform \"foo\", found in:") }) @@ -136,7 +137,7 @@ func TestValidateUniqueFields(t *testing.T) { `), 0o644) require.NoError(t, err) - errs := ValidateUniqueFields(fspath.DirFS(d)) + errs := ValidateUniqueFields(pkgpath.NewCachedFS(fspath.DirFS(d))) require.Len(t, errs, 1) assert.ErrorContains(t, errs[0], "field \"field2\" is defined multiple times, found in:") }) diff --git a/code/go/internal/validator/semantic/validate_var_groups.go b/code/go/internal/validator/semantic/validate_var_groups.go index 07d28ef36..bdaccd0d5 100644 --- a/code/go/internal/validator/semantic/validate_var_groups.go +++ b/code/go/internal/validator/semantic/validate_var_groups.go @@ -6,12 +6,10 @@ package semantic import ( "fmt" - "io/fs" "path" "gopkg.in/yaml.v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -21,9 +19,16 @@ import ( // - var_group names are unique // - option names within each var_group are unique // - vars in a var_group must not have required: true (requirement is controlled by var_group) -func ValidateVarGroups(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateVarGroups(fsys PackageFS) specerrors.ValidationErrors { // Validate main manifest. - d, err := fs.ReadFile(fsys, "manifest.yml") + files, err := fsys.Files("manifest.yml") + if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("failed to read file \"%s\": %w", fsys.Path("manifest.yml"), err)} + } + if len(files) == 0 { + return nil + } + d, err := files[0].ReadAll() if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("failed to read file \"%s\": %w", fsys.Path("manifest.yml"), err)} } @@ -103,12 +108,19 @@ func validateVarGroupsManifest(filePath string, manifest varGroupsManifest) spec return errs } -func validateDataStreamVarGroups(fsys fspath.FS, filePath string, pkgManifest varGroupsManifest) specerrors.ValidationErrors { - d, err := fs.ReadFile(fsys, filePath) +func validateDataStreamVarGroups(fsys PackageFS, filePath string, pkgManifest varGroupsManifest) specerrors.ValidationErrors { + files, err := fsys.Files(filePath) if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("failed to read file \"%s\": %w", fsys.Path(filePath), err)} + } + if len(files) == 0 { // File might not exist, which is fine return nil } + d, err := files[0].ReadAll() + if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("failed to read file \"%s\": %w", fsys.Path(filePath), err)} + } var manifest varGroupsDataStreamManifest err = yaml.Unmarshal(d, &manifest) diff --git a/code/go/internal/validator/semantic/validate_version_integrity.go b/code/go/internal/validator/semantic/validate_version_integrity.go index 5806571d9..d75f59758 100644 --- a/code/go/internal/validator/semantic/validate_version_integrity.go +++ b/code/go/internal/validator/semantic/validate_version_integrity.go @@ -11,14 +11,12 @@ import ( "github.com/Masterminds/semver/v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" - "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) // ValidateVersionIntegrity returns validation errors if the version defined in manifest isn't referenced in the latest // entry of the changelog file. -func ValidateVersionIntegrity(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateVersionIntegrity(fsys PackageFS) specerrors.ValidationErrors { manifestVersion, err := readManifestVersion(fsys) if err != nil { return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} @@ -46,7 +44,7 @@ func ValidateVersionIntegrity(fsys fspath.FS) specerrors.ValidationErrors { return nil } -func readManifestVersion(fsys fspath.FS) (string, error) { +func readManifestVersion(fsys PackageFS) (string, error) { manifest, err := readManifest(fsys) if err != nil { return "", err @@ -64,13 +62,13 @@ func readManifestVersion(fsys fspath.FS) (string, error) { return sVal, nil } -func readChangelogVersions(fsys fspath.FS) ([]string, error) { +func readChangelogVersions(fsys PackageFS) ([]string, error) { return readChangelog(fsys, "$[*].version") } -func readChangelog(fsys fspath.FS, jsonpath string) ([]string, error) { +func readChangelog(fsys PackageFS, jsonpath string) ([]string, error) { changelogPath := "changelog.yml" - f, err := pkgpath.Files(fsys, changelogPath) + f, err := fsys.Files(changelogPath) if err != nil { return nil, fmt.Errorf("can't locate changelog file: %w", err) } diff --git a/code/go/internal/validator/semantic/validate_visualizations_used_by_value.go b/code/go/internal/validator/semantic/validate_visualizations_used_by_value.go index a50d7134a..5481a614f 100644 --- a/code/go/internal/validator/semantic/validate_visualizations_used_by_value.go +++ b/code/go/internal/validator/semantic/validate_visualizations_used_by_value.go @@ -9,8 +9,6 @@ import ( "fmt" "path" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" - "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -25,11 +23,11 @@ type reference struct { // That is, it warns if a Kibana dashbaord file, foo.json, // defines some visualization using reference (containing an element of // "visualization" type inside references key). -func ValidateVisualizationsUsedByValue(fsys fspath.FS) specerrors.ValidationErrors { +func ValidateVisualizationsUsedByValue(fsys PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors filePaths := path.Join("kibana", "dashboard", "*.json") - objectFiles, err := pkgpath.Files(fsys, filePaths) + objectFiles, err := fsys.Files(filePaths) if err != nil { errs = append(errs, specerrors.NewStructuredErrorf("error finding Kibana Dashboard files: %w", err)) return errs diff --git a/code/go/internal/validator/semantic/warning.go b/code/go/internal/validator/semantic/warning.go index 09cf76a70..022266ed0 100644 --- a/code/go/internal/validator/semantic/warning.go +++ b/code/go/internal/validator/semantic/warning.go @@ -7,7 +7,6 @@ package semantic import ( "log" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/validator/common" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -15,8 +14,8 @@ import ( // WarnOn returns a validation function that wraps another one. Errors returned by the // wrapped validation that have a filtering code are printed as warnings. Other errors // are directly returned. -func WarnOn(validation func(fsys fspath.FS) specerrors.ValidationErrors) func(fspath.FS) specerrors.ValidationErrors { - return func(fsys fspath.FS) specerrors.ValidationErrors { +func WarnOn(validation func(fsys PackageFS) specerrors.ValidationErrors) func(PackageFS) specerrors.ValidationErrors { + return func(fsys PackageFS) specerrors.ValidationErrors { errs := validation(fsys) if common.IsDefinedWarningsAsErrors() { return errs diff --git a/code/go/internal/validator/spec.go b/code/go/internal/validator/spec.go index ce7bdf1f9..0ba47cace 100644 --- a/code/go/internal/validator/spec.go +++ b/code/go/internal/validator/spec.go @@ -16,9 +16,9 @@ import ( "github.com/Masterminds/semver/v3" spec "github.com/elastic/package-spec/v3" - "github.com/elastic/package-spec/v3/code/go/internal/fspath" "github.com/elastic/package-spec/v3/code/go/internal/loader" "github.com/elastic/package-spec/v3/code/go/internal/packages" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/internal/spectypes" "github.com/elastic/package-spec/v3/code/go/internal/validator/semantic" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" @@ -34,7 +34,7 @@ type Spec struct { fs fs.FS } -type validationRule func(pkg fspath.FS) specerrors.ValidationErrors +type validationRule func(pkg semantic.PackageFS) specerrors.ValidationErrors type validationRules []validationRule @@ -88,7 +88,7 @@ func (s Spec) ValidatePackage(pkg packages.Package) specerrors.ValidationErrors errs = append(errs, validator.Validate()...) // Semantic validations - errs = append(errs, s.rules(pkg.Type, rootSpec).validate(&pkg)...) + errs = append(errs, s.rules(pkg.Type, rootSpec).validate(pkgpath.NewCachedFS(&pkg))...) return processErrors(errs) } @@ -255,7 +255,7 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules return validationRules } -func (vr validationRules) validate(fsys fspath.FS) specerrors.ValidationErrors { +func (vr validationRules) validate(fsys semantic.PackageFS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors for _, validationRule := range vr { err := validationRule(fsys)