diff --git a/code/go/internal/specschema/folder_item_spec.go b/code/go/internal/specschema/folder_item_spec.go index 70472cdd7..b9a61514b 100644 --- a/code/go/internal/specschema/folder_item_spec.go +++ b/code/go/internal/specschema/folder_item_spec.go @@ -81,6 +81,12 @@ func (s *ItemSpec) DevelopmentFolder() bool { return s.itemSpec.DevelopmentFolder } +// ValidationMode returns the mode in which this item is valid: "source", +// "build", or "" (both modes). +func (s *ItemSpec) ValidationMode() string { + return s.itemSpec.ValidationMode +} + // AllowLink returns true if the item allows links. func (s *ItemSpec) AllowLink() bool { return s.itemSpec.AllowLink @@ -142,6 +148,10 @@ type folderItemSpec struct { DevelopmentFolder bool `json:"developmentFolder" yaml:"developmentFolder"` AllowLink bool `json:"allowLink" yaml:"allowLink"` + // ValidationMode restricts the item to a specific validation mode: "source", + // "build", or "" (both modes). + ValidationMode string `json:"validationMode" yaml:"validationMode"` + // As it is required to be inline both in yaml and json, this struct must be public embedded field SpecLimits `yaml:",inline"` diff --git a/code/go/internal/specschema/folder_spec.go b/code/go/internal/specschema/folder_spec.go index 8a2451a7c..9015fc270 100644 --- a/code/go/internal/specschema/folder_spec.go +++ b/code/go/internal/specschema/folder_spec.go @@ -97,6 +97,10 @@ func (l *FolderSpecLoader) loadContents(s *folderItemSpec, fs fs.FS, specPath st return fmt.Errorf("item [%s] visibility is expected to be private or public, not [%s]", path.Join(specPath, content.Name), content.Visibility) } + if vm := content.ValidationMode; vm != "" && vm != spectypes.ValidationModeSource && vm != spectypes.ValidationModeBuild { + return fmt.Errorf("item [%s] validationMode is expected to be %q or %q, not [%s]", path.Join(specPath, content.Name), spectypes.ValidationModeSource, spectypes.ValidationModeBuild, vm) + } + // All folders inside a development folder are too. if s.DevelopmentFolder { content.DevelopmentFolder = true diff --git a/code/go/internal/spectypes/item.go b/code/go/internal/spectypes/item.go index fee260b8e..823caeee3 100644 --- a/code/go/internal/spectypes/item.go +++ b/code/go/internal/spectypes/item.go @@ -16,6 +16,14 @@ const ( // ItemTypeFolder is the type of an item that represents a folder. ItemTypeFolder = "folder" + + // ValidationModeSource marks an item as valid only in source packages (not in built packages). + // The string value must match validator.SourceMode. + ValidationModeSource = "source" + + // ValidationModeBuild marks an item as valid only in built packages (not in source packages). + // The string value must match validator.BuildMode. + ValidationModeBuild = "build" ) // LimitsSpec contain the specifications related to limits. @@ -57,6 +65,10 @@ type ItemSpec interface { // DevelopmentFolder returns true if the item is inside a development folder. DevelopmentFolder() bool + // ValidationMode returns the mode in which this item is valid: "source", + // "build", or "" (both modes). + ValidationMode() string + // AllowLink returns true if the item allows links. AllowLink() bool diff --git a/code/go/internal/validator/folder_item_spec.go b/code/go/internal/validator/folder_item_spec.go index bca7cb808..b1a37d186 100644 --- a/code/go/internal/validator/folder_item_spec.go +++ b/code/go/internal/validator/folder_item_spec.go @@ -13,6 +13,15 @@ import ( "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) +// itemForbiddenInMode returns true if the item is forbidden in the given mode. +// LegacyMode is never restricted. +func itemForbiddenInMode(spec spectypes.ItemSpec, mode Mode) bool { + if mode == LegacyMode { + return false + } + return spec.ValidationMode() != "" && spec.ValidationMode() != string(mode) +} + func matchingFileExists(spec spectypes.ItemSpec, files []fs.DirEntry) (bool, error) { if spec.Name() != "" { for _, file := range files { diff --git a/code/go/internal/validator/folder_spec.go b/code/go/internal/validator/folder_spec.go index 0d1c82d1c..a788d8aa9 100644 --- a/code/go/internal/validator/folder_spec.go +++ b/code/go/internal/validator/folder_spec.go @@ -23,21 +23,23 @@ type validator struct { pkg *packages.Package folderPath string warningsAsErrors bool + mode Mode totalSize spectypes.FileSize totalContents int } -func newValidator(spec spectypes.ItemSpec, pkg *packages.Package, warningsAsErrors bool) *validator { - return newValidatorForPath(spec, pkg, ".", warningsAsErrors) +func newValidator(spec spectypes.ItemSpec, pkg *packages.Package, warningsAsErrors bool, mode Mode) *validator { + return newValidatorForPath(spec, pkg, ".", warningsAsErrors, mode) } -func newValidatorForPath(spec spectypes.ItemSpec, pkg *packages.Package, folderPath string, warningsAsErrors bool) *validator { +func newValidatorForPath(spec spectypes.ItemSpec, pkg *packages.Package, folderPath string, warningsAsErrors bool, mode Mode) *validator { return &validator{ spec: spec, pkg: pkg, folderPath: folderPath, warningsAsErrors: warningsAsErrors, + mode: mode, } } @@ -83,6 +85,15 @@ func (v *validator) Validate() specerrors.ValidationErrors { for _, file := range files { fileName := file.Name() + + if isLink, _ := checkLink(fileName); v.mode == BuildMode && isLink { + errs = append(errs, specerrors.NewStructuredErrorf( + "file %q: .link files are not allowed in built packages", + v.pkg.Path(path.Join(v.folderPath, fileName)), + )) + continue + } + itemSpec, err := v.findItemSpec(fileName) if err != nil { errs = append(errs, specerrors.NewStructuredError(err, specerrors.UnassignedCode)) @@ -119,8 +130,18 @@ func (v *validator) Validate() specerrors.ValidationErrors { continue } + if itemForbiddenInMode(itemSpec, v.mode) { + errs = append(errs, specerrors.NewStructuredErrorf( + "file %q: %s-only folder is not allowed in %s packages", + v.pkg.Path(path.Join(v.folderPath, fileName)), + itemSpec.ValidationMode(), + string(v.mode), + )) + continue + } + subFolderPath := path.Join(v.folderPath, fileName) - itemValidator := newValidatorForPath(itemSpec, v.pkg, subFolderPath, v.warningsAsErrors) + itemValidator := newValidatorForPath(itemSpec, v.pkg, subFolderPath, v.warningsAsErrors, v.mode) subErrs := itemValidator.Validate() if len(subErrs) > 0 { errs = append(errs, subErrs...) @@ -139,6 +160,16 @@ func (v *validator) Validate() specerrors.ValidationErrors { continue } + if itemForbiddenInMode(itemSpec, v.mode) { + errs = append(errs, specerrors.NewStructuredErrorf( + "file %q: %s-only file is not allowed in %s packages", + v.pkg.Path(path.Join(v.folderPath, fileName)), + itemSpec.ValidationMode(), + string(v.mode), + )) + continue + } + itemPath := path.Join(v.folderPath, file.Name()) itemValidationErrs := validateFile(itemSpec, v.pkg, itemPath) for _, ive := range itemValidationErrs { @@ -231,9 +262,6 @@ func (v *validator) findItemSpec(folderItemName string) (spectypes.ItemSpec, err // checkLink checks if an item is a link and returns the item name without the // ".link" suffix if it is a link. func checkLink(itemName string) (bool, string) { - const linkExtension = ".link" - if strings.HasSuffix(itemName, linkExtension) { - return true, strings.TrimSuffix(itemName, linkExtension) - } - return false, itemName + stripped, isLink := strings.CutSuffix(itemName, ".link") + return isLink, stripped } diff --git a/code/go/internal/validator/modes.go b/code/go/internal/validator/modes.go index 70ebc4444..4fa20e9cb 100644 --- a/code/go/internal/validator/modes.go +++ b/code/go/internal/validator/modes.go @@ -14,9 +14,11 @@ const ( LegacyMode Mode = "legacy" // SourceMode validates a package as a checked-out source tree: linked files // are resolved transparently and source-only rules are enforced. + // The string value must match spectypes.ValidationModeSource. SourceMode Mode = "source" // BuildMode validates a package as a built artifact: linked files are // unconditionally blocked and build-only rules are enforced. + // The string value must match spectypes.ValidationModeBuild. BuildMode Mode = "build" ) diff --git a/code/go/internal/validator/semantic/types.go b/code/go/internal/validator/semantic/types.go index 51a1fc5b7..c557ae600 100644 --- a/code/go/internal/validator/semantic/types.go +++ b/code/go/internal/validator/semantic/types.go @@ -352,9 +352,11 @@ func listDataStreams(fsys fspath.FS) ([]string, error) { return nil, fmt.Errorf("can't list data streams directory: %w", err) } - list := make([]string, len(dataStreams)) - for i, dataStream := range dataStreams { - list[i] = dataStream.Name() + var list []string + for _, dataStream := range dataStreams { + if dataStream.IsDir() { + list = append(list, dataStream.Name()) + } } return list, nil } diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories.go b/code/go/internal/validator/semantic/validate_datastream_package_categories.go index 0a4de7638..7e538b383 100644 --- a/code/go/internal/validator/semantic/validate_datastream_package_categories.go +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories.go @@ -105,7 +105,7 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} } - if pkgType != packageTypeIntegration { + if pkgType != integrationPackageType { return nil } diff --git a/code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go b/code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go index b393da6a4..09ca8eb34 100644 --- a/code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go +++ b/code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go @@ -47,7 +47,7 @@ func ValidateIntegrationInputsDeprecation(fsys fspath.FS) specerrors.ValidationE specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} } // skip if not an integration package - if m.Type != packageTypeIntegration { + if m.Type != integrationPackageType { return nil } diff --git a/code/go/internal/validator/semantic/validate_integration_policy_template_path.go b/code/go/internal/validator/semantic/validate_integration_policy_template_path.go index b7a93cc18..ea240ba5f 100644 --- a/code/go/internal/validator/semantic/validate_integration_policy_template_path.go +++ b/code/go/internal/validator/semantic/validate_integration_policy_template_path.go @@ -20,11 +20,11 @@ import ( const ( defaultStreamTemplatePath = "stream.yml.hbs" - packageTypeIntegration = "integration" ) type policyTemplateInput struct { Type string `yaml:"type"` + Package string `yaml:"package"` TemplatePath string `yaml:"template_path"` TemplatePaths []string `yaml:"template_paths"` } @@ -41,6 +41,7 @@ type integrationPackageManifest struct { // package manifest type stream struct { Input string `yaml:"input"` + Package string `yaml:"package"` TemplatePath string `yaml:"template_path"` TemplatePaths []string `yaml:"template_paths"` } @@ -78,7 +79,7 @@ func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErr specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)} } - if manifest.Type != packageTypeIntegration { + if manifest.Type != integrationPackageType { return nil } @@ -110,6 +111,9 @@ func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErr // under agent/input when template_paths or template_path is set (Fleet: template_paths first). func validateIntegrationPolicyTemplateInputs(fsys fspath.FS, policyTemplate integrationPolicyTemplate) error { for _, input := range policyTemplate.Inputs { + // Only validate template files that are explicitly declared; if none are set there + // is nothing to check (composable inputs without overlay templates source them from + // the dependency package, which is absent from the source tree). if len(input.TemplatePaths) > 0 { for _, tp := range input.TemplatePaths { if err := validateAgentInputTemplatePath(fsys, tp); err != nil { @@ -141,6 +145,11 @@ func validateAllDataStreamStreamTemplates(fsys fspath.FS, dsMap map[string]dataS dsManifestPath := path.Join(dsDir, "manifest.yml") manifest := dsMap[dsDir] for _, s := range manifest.Streams { + // Don't validate template paths if they use a package as input + // and they don't define any template. + if s.Package != "" && s.TemplatePath == "" && len(s.TemplatePaths) == 0 { + continue + } if err := validateSingleDataStreamStreamTemplates(fsys, dsDir, s); err != nil { errs = append(errs, specerrors.NewStructuredErrorf( "file \"%s\" is invalid: data stream \"%s\" stream input %q: %w", diff --git a/code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go b/code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go index 3c5fd7080..618c3aeae 100644 --- a/code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go +++ b/code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go @@ -254,6 +254,65 @@ streams: errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) require.Empty(t, errs) }) + + t.Run("composable stream with no explicit templates skips validation", func(t *testing.T) { + d := t.TempDir() + writeMinimalIntegrationManifest(t, d) + // No agent/stream directory — templates come entirely from the input package. + err := os.MkdirAll(filepath.Join(d, "data_stream", "logs"), 0o755) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "manifest.yml"), []byte(` +streams: + - package: some_input_package + title: Composable + description: d +`), 0o644) + require.NoError(t, err) + + errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) + require.Empty(t, errs) + }) + + t.Run("composable stream with explicit template_paths validates those files", func(t *testing.T) { + d := t.TempDir() + writeMinimalIntegrationManifest(t, d) + err := os.MkdirAll(filepath.Join(d, "data_stream", "logs", "agent", "stream"), 0o755) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", "overlay.yml.hbs"), []byte(`x`), 0o644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "manifest.yml"), []byte(` +streams: + - package: some_input_package + title: Composable with overlay + description: d + template_paths: + - overlay.yml.hbs +`), 0o644) + require.NoError(t, err) + + errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) + require.Empty(t, errs) + }) + + t.Run("composable stream with explicit template_paths fails when file missing", func(t *testing.T) { + d := t.TempDir() + writeMinimalIntegrationManifest(t, d) + err := os.MkdirAll(filepath.Join(d, "data_stream", "logs", "agent", "stream"), 0o755) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "manifest.yml"), []byte(` +streams: + - package: some_input_package + title: Composable with overlay + description: d + template_paths: + - missing.yml.hbs +`), 0o644) + require.NoError(t, err) + + errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) + require.Len(t, errs, 1) + require.Contains(t, errs[0].Error(), "template file not found") + }) } func TestValidateIntegrationPolicyTemplates_NonIntegrationType(t *testing.T) { d := t.TempDir() @@ -323,6 +382,70 @@ streams: errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) require.Empty(t, errs) } +func TestValidateIntegrationPolicyTemplates_ComposableInputs(t *testing.T) { + t.Run("composable input with no templates skips validation", func(t *testing.T) { + d := t.TempDir() + err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(` +type: integration +policy_templates: + - name: pt + inputs: + - package: some_input_package + title: Composable + description: d +`), 0o644) + require.NoError(t, err) + + errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) + require.Empty(t, errs) + }) + + t.Run("composable input with explicit template_paths validates those files", func(t *testing.T) { + d := t.TempDir() + err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "agent", "input", "overlay.yml.hbs"), []byte(`x`), 0o644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(` +type: integration +policy_templates: + - name: pt + inputs: + - package: some_input_package + title: Composable with overlay + description: d + template_paths: + - overlay.yml.hbs +`), 0o644) + require.NoError(t, err) + + errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) + require.Empty(t, errs) + }) + + t.Run("composable input with explicit template_paths fails when file missing", func(t *testing.T) { + d := t.TempDir() + err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(` +type: integration +policy_templates: + - name: pt + inputs: + - package: some_input_package + title: Composable with overlay + description: d + template_paths: + - missing.yml.hbs +`), 0o644) + require.NoError(t, err) + + errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d)) + require.Len(t, errs, 1) + require.Contains(t, errs[0].Error(), "template file not found") + }) +} + func TestFindPathAtDirectory(t *testing.T) { d := t.TempDir() diff --git a/code/go/internal/validator/semantic/validate_no_embedded_ecs.go b/code/go/internal/validator/semantic/validate_no_embedded_ecs.go new file mode 100644 index 000000000..fa03303c2 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_no_embedded_ecs.go @@ -0,0 +1,79 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "io/fs" + "path" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" +) + +// embeddedEcsDataStreamManifest is a minimal struct for unmarshalling just the +// elasticsearch.index_template.mappings.dynamic_templates section of a data stream manifest. +type embeddedEcsDataStreamManifest struct { + Elasticsearch struct { + IndexTemplate struct { + Mappings struct { + DynamicTemplates []map[string]any `yaml:"dynamic_templates"` + } `yaml:"mappings"` + } `yaml:"index_template"` + } `yaml:"elasticsearch"` +} + +// ValidateNoEmbeddedEcsInDynamicTemplates rejects data stream manifests that contain +// dynamic_templates entries whose keys match the "^_embedded_ecs" pattern. +// +// Keys starting with "_embedded_ecs" are auto-injected by elastic-package at build time +// when import_mappings is enabled. They must not appear in source packages and are +// rejected in source validation mode. Built packages (where these keys are expected) +// should be validated with ModeBuild, not ModeSource. +func ValidateNoEmbeddedEcsInDynamicTemplates(fsys fspath.FS) specerrors.ValidationErrors { + dataStreams, err := listDataStreams(fsys) + if err != nil { + return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)} + } + + var errs specerrors.ValidationErrors + for _, dataStream := range dataStreams { + manifestPath := path.Join(dataStreamDir, dataStream, "manifest.yml") + streamErrs := checkDataStreamManifestForEmbeddedEcs(fsys, manifestPath) + errs = append(errs, streamErrs...) + } + return errs +} + +func checkDataStreamManifestForEmbeddedEcs(fsys fspath.FS, manifestPath string) specerrors.ValidationErrors { + data, err := fs.ReadFile(fsys, manifestPath) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(manifestPath), err), + } + } + + var manifest embeddedEcsDataStreamManifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to parse manifest: %w", fsys.Path(manifestPath), err), + } + } + + var errs specerrors.ValidationErrors + for _, entry := range manifest.Elasticsearch.IndexTemplate.Mappings.DynamicTemplates { + for key := range entry { + if strings.HasPrefix(key, "_embedded_ecs") { + errs = append(errs, specerrors.NewStructuredErrorf( + "file \"%s\" is invalid: dynamic template %q starts with \"_embedded_ecs\"; this key is auto-injected at build time and must not appear in source packages", + fsys.Path(manifestPath), key, + )) + } + } + } + return errs +} diff --git a/code/go/internal/validator/semantic/validate_no_embedded_ecs_test.go b/code/go/internal/validator/semantic/validate_no_embedded_ecs_test.go new file mode 100644 index 000000000..f286f130f --- /dev/null +++ b/code/go/internal/validator/semantic/validate_no_embedded_ecs_test.go @@ -0,0 +1,189 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" +) + +func TestValidateNoEmbeddedEcsInDynamicTemplates(t *testing.T) { + tests := []struct { + name string + manifestYAML string + skipWriteStream bool // when true, data_stream/ is not created (simulates input/content packages) + expectErrors bool + errorContains []string + }{ + { + name: "no data_stream directory", + skipWriteStream: true, + expectErrors: false, + }, + { + name: "no dynamic_templates", + manifestYAML: ` +title: Test stream +type: logs +streams: + - input: logfile + title: Sample + description: Sample +`, + expectErrors: false, + }, + { + name: "dynamic_templates with regular names", + manifestYAML: ` +title: Test stream +type: logs +streams: + - input: logfile + title: Sample + description: Sample +elasticsearch: + index_template: + mappings: + dynamic_templates: + - my_ip_template: + mapping: + type: ip + match: ip + - my_date_template: + mapping: + type: date + match: timestamp +`, + expectErrors: false, + }, + { + name: "dynamic_templates with _embedded_ecs key", + manifestYAML: ` +title: Test stream +type: logs +streams: + - input: logfile + title: Sample + description: Sample +elasticsearch: + index_template: + mappings: + dynamic_templates: + - _embedded_ecs-ip_to_ip: + mapping: + type: ip + match: ip +`, + expectErrors: true, + errorContains: []string{"_embedded_ecs-ip_to_ip", "_embedded_ecs"}, + }, + { + name: "dynamic_templates mixed: regular and _embedded_ecs", + manifestYAML: ` +title: Test stream +type: logs +streams: + - input: logfile + title: Sample + description: Sample +elasticsearch: + index_template: + mappings: + dynamic_templates: + - my_ip_template: + mapping: + type: ip + match: ip + - _embedded_ecs-date_to_date: + mapping: + type: date + match: timestamp +`, + expectErrors: true, + errorContains: []string{"_embedded_ecs-date_to_date"}, + }, + { + name: "multiple _embedded_ecs entries", + manifestYAML: ` +title: Test stream +type: logs +streams: + - input: logfile + title: Sample + description: Sample +elasticsearch: + index_template: + mappings: + dynamic_templates: + - _embedded_ecs-ip_to_ip: + mapping: + type: ip + match: ip + - _embedded_ecs-port_to_long: + mapping: + type: long + match: port +`, + expectErrors: true, + errorContains: []string{"_embedded_ecs-ip_to_ip", "_embedded_ecs-port_to_long"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tempDir := t.TempDir() + + if !tc.skipWriteStream { + // Create data_stream/test_stream/manifest.yml + dsManifestDir := filepath.Join(tempDir, "data_stream", "test_stream") + require.NoError(t, os.MkdirAll(dsManifestDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dsManifestDir, "manifest.yml"), []byte(tc.manifestYAML), 0644)) + } + + fsys := fspath.DirFS(tempDir) + errs := ValidateNoEmbeddedEcsInDynamicTemplates(fsys) + + if !tc.expectErrors { + assert.Nil(t, errs, "expected no errors but got: %v", errs) + return + } + + require.NotNil(t, errs, "expected validation errors but got none") + var sb strings.Builder + for _, e := range errs { + sb.WriteString(e.Error()) + sb.WriteString("\n") + } + combined := sb.String() + for _, substr := range tc.errorContains { + assert.Contains(t, combined, substr) + } + }) + } +} + +// TestListDataStreamsIgnoresNonDirectories verifies that stray files directly +// under data_stream/ (e.g. .gitkeep, .DS_Store) are silently skipped and do +// not cause spurious validation errors. +func TestListDataStreamsIgnoresNonDirectories(t *testing.T) { + tempDir := t.TempDir() + + // Create data_stream/ with only a stray file — no subdirectories. + dataStreamDir := filepath.Join(tempDir, "data_stream") + require.NoError(t, os.MkdirAll(dataStreamDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dataStreamDir, ".gitkeep"), []byte(""), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dataStreamDir, ".DS_Store"), []byte(""), 0644)) + + fsys := fspath.DirFS(tempDir) + errs := ValidateNoEmbeddedEcsInDynamicTemplates(fsys) + assert.Nil(t, errs, "stray files under data_stream/ should produce no errors, got: %v", errs) +} diff --git a/code/go/internal/validator/semantic/validate_no_external_fields.go b/code/go/internal/validator/semantic/validate_no_external_fields.go new file mode 100644 index 000000000..c5ad99d4c --- /dev/null +++ b/code/go/internal/validator/semantic/validate_no_external_fields.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" +) + +// ValidateNoExternalFields errors for any field with an external field. +// The build process materializes ECS field references — once built, fields +// should carry full definitions, not external pointers. A built package must +// not contain any fields with external: ecs when validated with ModeBuild. +func ValidateNoExternalFields(fsys fspath.FS) specerrors.ValidationErrors { + validateFunc := func(metadata fieldFileMetadata, f field) specerrors.ValidationErrors { + if f.External == "" { + return nil + } + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf( + "file \"%s\" is invalid: field %s has external: %s reference; external fields must be materialized before packaging", + metadata.fullFilePath, f.Name, f.External, + ), + } + } + return validateFields(fsys, validateFunc) +} diff --git a/code/go/internal/validator/semantic/validate_no_external_fields_test.go b/code/go/internal/validator/semantic/validate_no_external_fields_test.go new file mode 100644 index 000000000..c79592b95 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_no_external_fields_test.go @@ -0,0 +1,96 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" +) + +func TestValidateNoExternalFields(t *testing.T) { + tests := []struct { + name string + files map[string]string // path → content + expectErrors bool + errorContains []string + }{ + { + name: "no external fields — no errors", + files: map[string]string{ + "data_stream/foo/fields/fields.yml": "- name: message\n type: keyword\n", + }, + expectErrors: false, + }, + { + name: "materialized ECS field (no external key) — no errors", + files: map[string]string{ + "data_stream/foo/fields/base-fields.yml": "- name: data_stream.type\n type: constant_keyword\n description: Data stream type.\n", + }, + expectErrors: false, + }, + { + name: "field with external: ecs — rejected", + files: map[string]string{ + "data_stream/foo/fields/ecs.yml": "- name: host.name\n external: ecs\n", + }, + expectErrors: true, + errorContains: []string{"host.name", "external: ecs", "external fields must be materialized"}, + }, + { + name: "multiple fields with external: ecs — all rejected", + files: map[string]string{ + "data_stream/foo/fields/ecs.yml": "- name: host.name\n external: ecs\n- name: agent.version\n external: ecs\n", + }, + expectErrors: true, + errorContains: []string{"external: ecs", "external fields must be materialized"}, + }, + { + name: "field with non-ecs external — rejected by this rule", + files: map[string]string{ + "data_stream/foo/fields/custom.yml": "- name: myfield\n external: custom_dep\n", + }, + expectErrors: true, + errorContains: []string{"external: custom_dep", "external fields must be materialized"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tempDir := t.TempDir() + + for relPath, content := range tc.files { + fullPath := filepath.Join(tempDir, relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0o755)) + require.NoError(t, os.WriteFile(fullPath, []byte(content), 0o644)) + } + + fsys := fspath.DirFS(tempDir) + errs := ValidateNoExternalFields(fsys) + + if !tc.expectErrors { + assert.Nil(t, errs, "expected no errors but got: %v", errs) + return + } + + require.NotNil(t, errs, "expected validation errors but got none") + var sb strings.Builder + for _, e := range errs { + sb.WriteString(e.Error()) + sb.WriteString("\n") + } + combined := sb.String() + for _, substr := range tc.errorContains { + assert.Contains(t, combined, substr) + } + }) + } +} diff --git a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go index dec6f3643..5d9088f34 100644 --- a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go +++ b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go @@ -70,7 +70,7 @@ func ValidatePolicyTemplateDatastreamCategories(fsys fspath.FS) specerrors.Valid } // only validate integration type packages - if pkgType != packageTypeIntegration { + if pkgType != integrationPackageType { return nil } diff --git a/code/go/internal/validator/semantic/validate_stream_input_bundled.go b/code/go/internal/validator/semantic/validate_stream_input_bundled.go new file mode 100644 index 000000000..f2d204124 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_stream_input_bundled.go @@ -0,0 +1,153 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "errors" + "io/fs" + "path" + "slices" + + "gopkg.in/yaml.v3" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" +) + +// streamMaterializationEntry captures the fields needed to check input materialization +// in a data stream manifest's streams[] array. +type streamMaterializationEntry struct { + Input string `yaml:"input"` + Package string `yaml:"package"` +} + +type dataStreamMaterializationManifest struct { + Streams []streamMaterializationEntry `yaml:"streams"` +} + +// policyTemplateInputMaterialization captures only the fields needed from a policy template input +// to check whether materialization has taken place. +type policyTemplateInputMaterialization struct { + Type string `yaml:"type"` + Package string `yaml:"package"` +} + +type policyTemplateMaterialization struct { + Name string `yaml:"name"` + Inputs []policyTemplateInputMaterialization `yaml:"inputs"` +} + +type packageMaterializationManifest struct { + Type string `yaml:"type"` + PolicyTemplates []policyTemplateMaterialization `yaml:"policy_templates"` +} + +// ValidateStreamInputBundled errors when build-mode manifests carry +// source-only 'package:' fields that the build process should have bundled: +// +// - data_stream/*/manifest.yml: each stream must have 'input:' set and must NOT +// have 'package:' (composable-input pattern, source-only). +// - manifest.yml: each policy_template input must have 'type:' set and must NOT +// have 'package:' (package-reference pattern, source-only). +func ValidateStreamInputBundled(fsys fspath.FS) specerrors.ValidationErrors { + var errs specerrors.ValidationErrors + + errs = append(errs, validateDataStreamStreamsBundled(fsys)...) + errs = append(errs, validatePolicyTemplateInputsBundled(fsys)...) + + return errs +} + +// validateDataStreamStreamsBundled checks every data_stream/*/manifest.yml for +// stream entries that carry a source-only 'package:' field or are missing 'input:'. +func validateDataStreamStreamsBundled(fsys fspath.FS) specerrors.ValidationErrors { + dataStreams, err := listDataStreams(fsys) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("can't list data streams: %w", err), + } + } + + // Sort for deterministic error ordering. + slices.Sort(dataStreams) + + var errs specerrors.ValidationErrors + for _, dataStreamName := range dataStreams { + manifestRelPath := path.Join(dataStreamDir, dataStreamName, "manifest.yml") + data, err := fs.ReadFile(fsys, manifestRelPath) + if err != nil { + errs = append(errs, specerrors.NewStructuredErrorf( + "file %q: failed to read data stream manifest: %w", + fsys.Path(manifestRelPath), err, + )) + continue + } + + var manifest dataStreamMaterializationManifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + errs = append(errs, specerrors.NewStructuredErrorf( + "file %q: failed to parse data stream manifest: %w", + fsys.Path(manifestRelPath), err, + )) + continue + } + + fullPath := fsys.Path(manifestRelPath) + for i, s := range manifest.Streams { + if s.Package != "" { + errs = append(errs, specerrors.NewStructuredErrorf( + "file %q: stream[%d] has 'package:' which is source-only; build packages must use 'input:' + 'template_paths:'", + fullPath, i, + )) + } + } + } + + return errs +} + +// validatePolicyTemplateInputsBundled checks the package-level manifest.yml for +// policy_template inputs that carry a source-only 'package:' field instead of 'type:'. +func validatePolicyTemplateInputsBundled(fsys fspath.FS) specerrors.ValidationErrors { + manifestRelPath := "manifest.yml" + data, err := fs.ReadFile(fsys, manifestRelPath) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("reading %q: %w", fsys.Path(manifestRelPath), err), + } + } + + var manifest packageMaterializationManifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf( + "file %q: failed to parse manifest: %w", + fsys.Path(manifestRelPath), err, + ), + } + } + + if manifest.Type != integrationPackageType { + return nil + } + + fullPath := fsys.Path(manifestRelPath) + var errs specerrors.ValidationErrors + for _, policyTemplate := range manifest.PolicyTemplates { + for i, input := range policyTemplate.Inputs { + if input.Package != "" { + errs = append(errs, specerrors.NewStructuredErrorf( + "file %q: policy_template %q input[%d] has 'package:' which is source-only; build packages must use 'type:'", + fullPath, policyTemplate.Name, i, + )) + } + } + } + + return errs +} diff --git a/code/go/internal/validator/semantic/validate_stream_input_bundled_test.go b/code/go/internal/validator/semantic/validate_stream_input_bundled_test.go new file mode 100644 index 000000000..830bddb97 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_stream_input_bundled_test.go @@ -0,0 +1,155 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" +) + +func TestValidateStreamInputBundled(t *testing.T) { + tests := []struct { + name string + // files maps relative path → YAML content written under a temp dir. + files map[string]string + expectErrors bool + errorContains []string + }{ + // --------------------------------------------------------------- + // Data stream manifest: happy paths + // --------------------------------------------------------------- + { + name: "data stream with input: set — no errors", + files: map[string]string{ + "data_stream/logs/manifest.yml": "title: Logs\ntype: logs\nstreams:\n - input: logfile\n title: Logs\n description: Collect logs\n", + }, + expectErrors: false, + }, + { + name: "no data_stream directory — no errors", + files: map[string]string{ + "manifest.yml": "type: integration\n", + }, + expectErrors: false, + }, + { + name: "data stream manifest with no streams array — no errors", + files: map[string]string{ + "data_stream/logs/manifest.yml": "title: Logs\ntype: logs\n", + }, + expectErrors: false, + }, + // --------------------------------------------------------------- + // Data stream manifest: error cases + // --------------------------------------------------------------- + { + name: "data stream stream has package: — rejected", + files: map[string]string{ + "data_stream/logs/manifest.yml": "title: Logs\ntype: logs\nstreams:\n - package: filelog_otel\n title: Logs\n description: Collect logs\n", + }, + expectErrors: true, + errorContains: []string{ + "stream[0]", + "'package:'", + "source-only", + "build packages must use 'input:'", + }, + }, + { + // Schema validation (oneOf: input|package) catches this before the + // semantic layer runs; the semantic check only looks for 'package:'. + name: "data stream stream missing input: — not caught by semantic layer", + files: map[string]string{ + "data_stream/logs/manifest.yml": "title: Logs\ntype: logs\nstreams:\n - title: Logs\n description: Collect logs\n", + }, + expectErrors: false, + }, + { + name: "multiple data streams, one bad — rejected", + files: map[string]string{ + "data_stream/good/manifest.yml": "title: Good\ntype: logs\nstreams:\n - input: logfile\n title: Good\n description: Good\n", + "data_stream/bad/manifest.yml": "title: Bad\ntype: logs\nstreams:\n - package: some_package\n title: Bad\n description: Bad\n", + }, + expectErrors: true, + errorContains: []string{ + "'package:'", + "source-only", + }, + }, + // --------------------------------------------------------------- + // Package manifest policy_templates: happy paths + // --------------------------------------------------------------- + { + name: "policy_template input with type: set — no errors", + files: map[string]string{ + "manifest.yml": "type: integration\npolicy_templates:\n - name: logs\n title: Logs\n description: Logs\n inputs:\n - type: logfile\n title: Logs\n description: Logs\n", + }, + expectErrors: false, + }, + { + name: "non-integration package manifest — no errors", + files: map[string]string{ + "manifest.yml": "type: input\npolicy_templates:\n - name: logs\n title: Logs\n description: Logs\n inputs:\n - package: some_package\n title: Logs\n description: Logs\n", + }, + expectErrors: false, + }, + // --------------------------------------------------------------- + // Package manifest policy_templates: error cases + // --------------------------------------------------------------- + { + name: "policy_template input has package: — rejected", + files: map[string]string{ + "manifest.yml": "type: integration\npolicy_templates:\n - name: events\n title: Events\n description: Events\n inputs:\n - package: filelog_otel\n title: Collect events\n description: Collecting events\n", + }, + expectErrors: true, + errorContains: []string{ + "policy_template", + "events", + "input[0]", + "'package:'", + "source-only", + "build packages must use 'type:'", + }, + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + tempDir := t.TempDir() + + for relPath, content := range testCase.files { + fullPath := filepath.Join(tempDir, relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0o755)) + require.NoError(t, os.WriteFile(fullPath, []byte(content), 0o644)) + } + + fsys := fspath.DirFS(tempDir) + errs := ValidateStreamInputBundled(fsys) + + if !testCase.expectErrors { + assert.Nil(t, errs, "expected no errors but got: %v", errs) + return + } + + require.NotNil(t, errs, "expected validation errors but got none") + var sb strings.Builder + for _, e := range errs { + sb.WriteString(e.Error()) + sb.WriteString("\n") + } + combined := sb.String() + for _, substr := range testCase.errorContains { + assert.Contains(t, combined, substr) + } + }) + } +} diff --git a/code/go/internal/validator/spec.go b/code/go/internal/validator/spec.go index 66bce2f99..35a01f09f 100644 --- a/code/go/internal/validator/spec.go +++ b/code/go/internal/validator/spec.go @@ -97,7 +97,7 @@ func (s Spec) ValidatePackage(pkg packages.Package) specerrors.ValidationErrors } // Syntactic validations - validator := newValidator(rootSpec, &pkg, s.WarningsAsErrors) + validator := newValidator(rootSpec, &pkg, s.WarningsAsErrors, s.mode) errs = append(errs, validator.Validate()...) // Semantic validations @@ -222,7 +222,8 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules {fn: semantic.ValidateDimensionFields, types: []string{"integration", "input"}}, {fn: semantic.ValidateDateFields, types: []string{"integration", "input"}}, {fn: semantic.ValidateRequiredFields, types: []string{"integration", "input"}}, - {fn: semantic.ValidateExternalFieldsWithDevFolder, types: []string{"integration", "input"}}, + {fn: semantic.ValidateExternalFieldsWithDevFolder, types: []string{"integration", "input"}, + modes: []Mode{LegacyMode, SourceMode}}, {fn: warnOn(semantic.ValidateVisualizationsUsedByValue), types: []string{"integration", "content"}, until: semver.MustParse("3.0.0")}, {fn: semantic.ValidateVisualizationsUsedByValue, types: []string{"integration", "content"}, since: semver.MustParse("3.0.0")}, {fn: semantic.ValidateILMPolicyPresent, since: semver.MustParse("2.0.0"), types: []string{"integration"}}, @@ -252,10 +253,17 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules {fn: semantic.ValidateKibanaTagDuplicates}, {fn: semantic.ValidatePipelineOnFailure, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, {fn: semantic.ValidateIntegrationInputsDeprecation, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, - {fn: semantic.ValidateIntegrationInputQualifier, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, + {fn: semantic.ValidateIntegrationInputQualifier, types: []string{"integration"}, since: semver.MustParse("3.6.0"), + modes: []Mode{LegacyMode, BuildMode}}, {fn: semantic.ValidateDeprecatedReplacedBy, since: semver.MustParse("3.6.0")}, {fn: semantic.ValidatePackageReferences, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, - {fn: semantic.ValidateTestPackageRequirements, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, + {fn: semantic.ValidateTestPackageRequirements, types: []string{"integration"}, since: semver.MustParse("3.6.0"), + modes: []Mode{LegacyMode, SourceMode}}, + {fn: semantic.ValidateNoEmbeddedEcsInDynamicTemplates, types: []string{"integration"}, + modes: []Mode{SourceMode}}, + {fn: semantic.ValidateNoExternalFields, modes: []Mode{BuildMode}}, + {fn: semantic.ValidateStreamInputBundled, modes: []Mode{BuildMode}, + types: []string{"integration"}}, } var validationRules validationRules diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index 0d6658c36..681205da8 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -23,7 +23,9 @@ import ( "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) -func TestValidateFile(t *testing.T) { +// Test_ValidateFromPath tests the ValidateFromPath function +// using the legacy specification. +func Test_ValidateFromPath(t *testing.T) { // Workaround for error messages that contain OS-dependant base paths. osTestBasePath := filepath.Join("..", "..", "..", "..", "test", "packages") + string(filepath.Separator) @@ -1265,6 +1267,9 @@ func TestValidateHandlebarsFiles(t *testing.T) { } } +// requireErrorMessage is a helper function to validate the error messages +// for the given package name and invalid items per folder. +// It uses the legacy specification. func requireErrorMessage(t *testing.T, pkgName string, invalidItemsPerFolder map[string][]string, expectedErrorMessage string) { pkgRootPath := filepath.Join("..", "..", "..", "..", "test", "packages", pkgName) @@ -1301,7 +1306,7 @@ func TestLinksBehaviorAcrossModes(t *testing.T) { require.Error(t, err) errs, ok := err.(specerrors.ValidationErrors) require.True(t, ok) - require.ErrorContains(t, errs, linkedfiles.ErrUnsupportedLinkFile.Error()) + require.ErrorContains(t, errs, ".link files are not allowed in built packages") }) t.Run("source_accepts_link_files", func(t *testing.T) { @@ -1480,9 +1485,13 @@ func TestValidateFromZip_modeRestrictions(t *testing.T) { t.Run("build_allowed", func(t *testing.T) { t.Parallel() + // Use the built fixture: source packages have _dev/ and external:ecs which + // build mode rejects. good_built is the correct built-package counterpart. + builtPkg := filepath.Join("..", "..", "..", "..", "test", "built_packages", "good_built") + builtZipPath := writePackageZip(t, builtPkg, "good_built") v, err := New(BuildMode) require.NoError(t, err) - err = v.ValidateFromZip(zipPath) + err = v.ValidateFromZip(builtZipPath) require.NoError(t, err) }) } @@ -1548,3 +1557,72 @@ func TestWithWarningsAsErrors_option(t *testing.T) { require.NoError(t, err) }) } + +func TestBuildModeValidation(t *testing.T) { + basePath := filepath.Join("..", "..", "..", "..", "test", "built_packages") + tests := map[string]struct { + expectedErrContains []string + }{ + "good_built": {}, + "bad_built_external_ecs": { + expectedErrContains: []string{"has external: ecs reference"}, + }, + "bad_built_missing_input": { + // Caught by schema oneOf (input|package), not the semantic layer. + expectedErrContains: []string{"streams.0: input is required"}, + }, + "bad_built_stream_package": { + expectedErrContains: []string{"stream[0] has 'package:' which is source-only"}, + }, + "bad_built_policy_template_package": { + expectedErrContains: []string{"input[0] has 'package:' which is source-only"}, + }, + "bad_built_fs_artifacts": { + expectedErrContains: []string{ + "source-only folder is not allowed in build packages", + ".link files are not allowed in built packages", + }, + }, + } + + for packageName, testCase := range tests { + t.Run(packageName, func(t *testing.T) { + t.Parallel() + v, err := New(BuildMode) + require.NoError(t, err) + err = v.ValidateFromPath(filepath.Join(basePath, packageName)) + if len(testCase.expectedErrContains) == 0 { + require.NoError(t, err) + return + } + require.Error(t, err) + for _, expectedError := range testCase.expectedErrContains { + require.ErrorContains(t, err, expectedError) + } + }) + } +} + +// TestSourceMode_BadEmbeddedEcs verifies that ValidateNoEmbeddedEcsInDynamicTemplates +// rejects the bad_embedded_ecs fixture when the validator runs in source mode. +// See test/packages/bad_embedded_ecs/data_stream/logs/manifest.yml for fixture details. +func TestSourceMode_BadEmbeddedEcs(t *testing.T) { + pkgPath := filepath.Join("..", "..", "..", "..", "test", "packages", "bad_embedded_ecs") + v, err := New(SourceMode) + require.NoError(t, err) + err = v.ValidateFromPath(pkgPath) + require.Error(t, err) + require.ErrorContains(t, err, "_embedded_ecs") +} + +// TestLegacyPreservation_FromPath verifies that the bad_embedded_ecs fixture passes +// in legacy mode: the spec schema permits _embedded_ecs keys in dynamic_templates for +// built packages, and legacy mode does not run source-only semantic rules. +// See test/packages/bad_embedded_ecs/data_stream/logs/manifest.yml for fixture details. +func TestLegacyPreservation_FromPath(t *testing.T) { + pkgPath := filepath.Join("..", "..", "..", "..", "test", "packages", "bad_embedded_ecs") + v, err := New(LegacyMode) + require.NoError(t, err) + err = v.ValidateFromPath(pkgPath) + require.NoError(t, err) +} diff --git a/spec/changelog.yml b/spec/changelog.yml index 47e03337f..968c3ec35 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -11,6 +11,9 @@ - description: Add support for mode-aware constructors and validation APIs. type: enhancement link: https://github.com/elastic/package-spec/pull/1177 + - description: Implement `source` and `build` validation modes. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1178 - version: 3.6.4-next changes: - description: Add provider_permissions field to package, policy_template, input, and data_stream levels for declaring provider-specific permissions. diff --git a/spec/content/spec.yml b/spec/content/spec.yml index e5dd5950c..ef9ff5c0a 100644 --- a/spec/content/spec.yml +++ b/spec/content/spec.yml @@ -61,6 +61,7 @@ spec: name: _dev required: false visibility: private + validationMode: source $ref: "./_dev/spec.yml" versions: diff --git a/spec/input/spec.yml b/spec/input/spec.yml index 9ccefea4c..661c3921f 100644 --- a/spec/input/spec.yml +++ b/spec/input/spec.yml @@ -55,6 +55,7 @@ spec: name: _dev required: false visibility: private + validationMode: source $ref: "./_dev/spec.yml" - description: File containing lifecycle configuration (technical preview) type: file diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index 9e3ff54ac..6d42551ff 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -175,6 +175,7 @@ spec: $ref: "../../integration/manifest.spec.yml#/definitions/deprecated" allOf: - if: + required: [type] properties: type: const: select @@ -217,13 +218,16 @@ spec: properties: name: pattern: "(_file|_url)$" - - properties: - type: - const: password + - allOf: + - required: [type] + - properties: + type: + const: password then: required: - secret - if: + required: [type] properties: type: const: duration @@ -234,9 +238,11 @@ spec: pattern: '^(\d+[smh]|\d+ms)+$' - if: not: - properties: - type: - const: duration + allOf: + - required: [type] + - properties: + type: + const: duration then: not: anyOf: @@ -348,7 +354,8 @@ spec: $ref: "./fields/fields.spec.yml#/items/properties/index" patternProperties: # Exception for fields imported by elastic-package when import_mappings is used. - # TODO: Allow this only on built packages. + # Source packages: rejected by ValidateNoEmbeddedEcsInDynamicTemplates (issue #549). + # Built packages: permitted (the key is auto-injected at build time). "^_embedded_ecs": type: object additionalProperties: false @@ -640,7 +647,14 @@ spec: required_vars: $ref: "#/definitions/required_vars" vars: - $ref: "#/definitions/vars" + # Base schema is permissive: composable streams ('package:') carry + # partial var overrides where name and type come from the dependency + # and are only present in the built package. + # Full validation ($ref: "#/definitions/vars") is applied below via + # if/then for non-composable streams ('input:'). + type: array + items: + type: object var_groups: $ref: "../../integration/manifest.spec.yml#/definitions/var_groups" sections: @@ -658,6 +672,12 @@ spec: oneOf: - required: [input] - required: [package] + if: + required: [input] + then: + properties: + vars: + $ref: "#/definitions/vars" agent: $ref: "../../integration/manifest.spec.yml#/definitions/agent" elasticsearch: diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 5172b857b..e42bab8e8 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -84,6 +84,7 @@ spec: name: _dev required: false visibility: private + validationMode: source $ref: "./_dev/spec.yml" - description: File containing routing rules definitions (technical preview) type: file diff --git a/spec/integration/manifest.spec.yml b/spec/integration/manifest.spec.yml index 6389bb589..0c0bb9d88 100644 --- a/spec/integration/manifest.spec.yml +++ b/spec/integration/manifest.spec.yml @@ -961,7 +961,14 @@ spec: required_vars: $ref: "./data_stream/manifest.spec.yml#/definitions/required_vars" vars: - $ref: "./data_stream/manifest.spec.yml#/definitions/vars" + # Base schema is permissive: composable inputs ('package:') carry + # partial var overrides; name and type come from the dependency + # and are only materialised in the built package. + # Full validation is applied below via if/then for non-composable + # inputs ('type:'). + type: array + items: + type: object var_groups: $ref: "#/definitions/var_groups" sections: @@ -991,6 +998,12 @@ spec: oneOf: - required: [type] - required: [package] + if: + required: [type] + then: + properties: + vars: + $ref: "./data_stream/manifest.spec.yml#/definitions/vars" multiple: type: boolean icons: diff --git a/spec/integration/spec.yml b/spec/integration/spec.yml index eb46859e2..66b241704 100644 --- a/spec/integration/spec.yml +++ b/spec/integration/spec.yml @@ -60,6 +60,7 @@ spec: name: _dev required: false visibility: private + validationMode: source $ref: "./_dev/spec.yml" - description: Folder containing Elasticsearch assets used by the package type: folder diff --git a/test/built_packages/bad_built_external_ecs/LICENSE.txt b/test/built_packages/bad_built_external_ecs/LICENSE.txt new file mode 100644 index 000000000..f6f788a89 --- /dev/null +++ b/test/built_packages/bad_built_external_ecs/LICENSE.txt @@ -0,0 +1,3 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license diff --git a/test/built_packages/bad_built_external_ecs/changelog.yml b/test/built_packages/bad_built_external_ecs/changelog.yml new file mode 100644 index 000000000..e00f88133 --- /dev/null +++ b/test/built_packages/bad_built_external_ecs/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 diff --git a/test/built_packages/bad_built_external_ecs/data_stream/events/agent/stream/stream.yml.hbs b/test/built_packages/bad_built_external_ecs/data_stream/events/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..9390bc05c --- /dev/null +++ b/test/built_packages/bad_built_external_ecs/data_stream/events/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} diff --git a/test/built_packages/bad_built_external_ecs/data_stream/events/fields/base-fields.yml b/test/built_packages/bad_built_external_ecs/data_stream/events/fields/base-fields.yml new file mode 100644 index 000000000..0d1791ffe --- /dev/null +++ b/test/built_packages/bad_built_external_ecs/data_stream/events/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/built_packages/bad_built_external_ecs/data_stream/events/fields/fields.yml b/test/built_packages/bad_built_external_ecs/data_stream/events/fields/fields.yml new file mode 100644 index 000000000..e09745251 --- /dev/null +++ b/test/built_packages/bad_built_external_ecs/data_stream/events/fields/fields.yml @@ -0,0 +1,5 @@ +- name: message + type: keyword + description: Log message. +- name: host.name + external: ecs diff --git a/test/built_packages/bad_built_external_ecs/data_stream/events/manifest.yml b/test/built_packages/bad_built_external_ecs/data_stream/events/manifest.yml new file mode 100644 index 000000000..d51b2bd1c --- /dev/null +++ b/test/built_packages/bad_built_external_ecs/data_stream/events/manifest.yml @@ -0,0 +1,15 @@ +title: Events +type: logs +streams: + - input: logfile + title: Events logs + description: Collect events log data + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/*.log diff --git a/test/built_packages/bad_built_external_ecs/docs/README.md b/test/built_packages/bad_built_external_ecs/docs/README.md new file mode 100644 index 000000000..3c649908a --- /dev/null +++ b/test/built_packages/bad_built_external_ecs/docs/README.md @@ -0,0 +1,5 @@ +# Bad Built Package - External ECS + +This test fixture contains `external: ecs` field references that must be rejected +in build mode (issue `#549`). Built packages must materialize all ECS fields rather +than reference them via `external: ecs`. diff --git a/test/built_packages/bad_built_external_ecs/manifest.yml b/test/built_packages/bad_built_external_ecs/manifest.yml new file mode 100644 index 000000000..9b10b6ed1 --- /dev/null +++ b/test/built_packages/bad_built_external_ecs/manifest.yml @@ -0,0 +1,23 @@ +format_version: 3.6.0 +name: bad_built_external_ecs +title: Bad Built Package - External ECS Fields +description: Built-package fixture with external ECS fields - invalid in build mode (issue #549). +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0 || ^9.0.0' +policy_templates: + - name: events + title: Events logs + description: Collect events data + inputs: + - type: logfile + title: Collect events logs + description: Collecting events log data + multi: false +owner: + github: elastic/foobar + type: community diff --git a/test/built_packages/bad_built_fs_artifacts/LICENSE.txt b/test/built_packages/bad_built_fs_artifacts/LICENSE.txt new file mode 100644 index 000000000..f6f788a89 --- /dev/null +++ b/test/built_packages/bad_built_fs_artifacts/LICENSE.txt @@ -0,0 +1,3 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license diff --git a/test/built_packages/bad_built_fs_artifacts/_dev/build/build.yml b/test/built_packages/bad_built_fs_artifacts/_dev/build/build.yml new file mode 100644 index 000000000..e69de29bb diff --git a/test/built_packages/bad_built_fs_artifacts/changelog.yml b/test/built_packages/bad_built_fs_artifacts/changelog.yml new file mode 100644 index 000000000..e00f88133 --- /dev/null +++ b/test/built_packages/bad_built_fs_artifacts/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 diff --git a/test/built_packages/bad_built_fs_artifacts/data_stream/events/_dev/test/.empty b/test/built_packages/bad_built_fs_artifacts/data_stream/events/_dev/test/.empty new file mode 100644 index 000000000..e69de29bb diff --git a/test/built_packages/bad_built_fs_artifacts/data_stream/events/agent/stream/stream.yml.hbs b/test/built_packages/bad_built_fs_artifacts/data_stream/events/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..9390bc05c --- /dev/null +++ b/test/built_packages/bad_built_fs_artifacts/data_stream/events/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} diff --git a/test/built_packages/bad_built_fs_artifacts/data_stream/events/fields/base-fields.yml b/test/built_packages/bad_built_fs_artifacts/data_stream/events/fields/base-fields.yml new file mode 100644 index 000000000..0d1791ffe --- /dev/null +++ b/test/built_packages/bad_built_fs_artifacts/data_stream/events/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/built_packages/bad_built_fs_artifacts/data_stream/events/fields/base-fields.yml.link b/test/built_packages/bad_built_fs_artifacts/data_stream/events/fields/base-fields.yml.link new file mode 100644 index 000000000..e69de29bb diff --git a/test/built_packages/bad_built_fs_artifacts/data_stream/events/fields/fields.yml b/test/built_packages/bad_built_fs_artifacts/data_stream/events/fields/fields.yml new file mode 100644 index 000000000..f21b45be3 --- /dev/null +++ b/test/built_packages/bad_built_fs_artifacts/data_stream/events/fields/fields.yml @@ -0,0 +1,3 @@ +- name: message + type: keyword + description: Log message. diff --git a/test/built_packages/bad_built_fs_artifacts/data_stream/events/manifest.yml b/test/built_packages/bad_built_fs_artifacts/data_stream/events/manifest.yml new file mode 100644 index 000000000..d51b2bd1c --- /dev/null +++ b/test/built_packages/bad_built_fs_artifacts/data_stream/events/manifest.yml @@ -0,0 +1,15 @@ +title: Events +type: logs +streams: + - input: logfile + title: Events logs + description: Collect events log data + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/*.log diff --git a/test/built_packages/bad_built_fs_artifacts/docs/README.md b/test/built_packages/bad_built_fs_artifacts/docs/README.md new file mode 100644 index 000000000..dfce52934 --- /dev/null +++ b/test/built_packages/bad_built_fs_artifacts/docs/README.md @@ -0,0 +1,6 @@ +# Bad Built Package - Source-only FS Artifacts + +This fixture contains both a `_dev/` directory and a `.link` file — both +source-only artifacts that must be rejected in build mode (issue `#549`). +Used to verify that `BuildMode` correctly flags packages containing either +`_dev/` directories or `.link` files. diff --git a/test/built_packages/bad_built_fs_artifacts/manifest.yml b/test/built_packages/bad_built_fs_artifacts/manifest.yml new file mode 100644 index 000000000..00840db0b --- /dev/null +++ b/test/built_packages/bad_built_fs_artifacts/manifest.yml @@ -0,0 +1,23 @@ +format_version: 3.6.0 +name: bad_built_fs_artifacts +title: Bad Built Package - Source-only FS Artifacts Present +description: Built-package fixture containing a _dev folder and a .link file - both source-only artifacts that must be rejected in build mode (issue #549). +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0 || ^9.0.0' +policy_templates: + - name: events + title: Events logs + description: Collect events data + inputs: + - type: logfile + title: Collect events logs + description: Collecting events log data + multi: false +owner: + github: elastic/foobar + type: community diff --git a/test/built_packages/bad_built_missing_input/LICENSE.txt b/test/built_packages/bad_built_missing_input/LICENSE.txt new file mode 100644 index 000000000..f6f788a89 --- /dev/null +++ b/test/built_packages/bad_built_missing_input/LICENSE.txt @@ -0,0 +1,3 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license diff --git a/test/built_packages/bad_built_missing_input/changelog.yml b/test/built_packages/bad_built_missing_input/changelog.yml new file mode 100644 index 000000000..e00f88133 --- /dev/null +++ b/test/built_packages/bad_built_missing_input/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 diff --git a/test/built_packages/bad_built_missing_input/data_stream/events/agent/stream/stream.yml.hbs b/test/built_packages/bad_built_missing_input/data_stream/events/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..9390bc05c --- /dev/null +++ b/test/built_packages/bad_built_missing_input/data_stream/events/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} diff --git a/test/built_packages/bad_built_missing_input/data_stream/events/fields/base-fields.yml b/test/built_packages/bad_built_missing_input/data_stream/events/fields/base-fields.yml new file mode 100644 index 000000000..0d1791ffe --- /dev/null +++ b/test/built_packages/bad_built_missing_input/data_stream/events/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/built_packages/bad_built_missing_input/data_stream/events/fields/fields.yml b/test/built_packages/bad_built_missing_input/data_stream/events/fields/fields.yml new file mode 100644 index 000000000..fa9de06d4 --- /dev/null +++ b/test/built_packages/bad_built_missing_input/data_stream/events/fields/fields.yml @@ -0,0 +1,5 @@ +- name: message + type: keyword + description: Log message. +- name: host.name + type: keyword diff --git a/test/built_packages/bad_built_missing_input/data_stream/events/manifest.yml b/test/built_packages/bad_built_missing_input/data_stream/events/manifest.yml new file mode 100644 index 000000000..ca8722e90 --- /dev/null +++ b/test/built_packages/bad_built_missing_input/data_stream/events/manifest.yml @@ -0,0 +1,5 @@ +title: Events +type: logs +streams: + - title: Events logs + description: Collect events log data (missing input — invalid in built packages) diff --git a/test/built_packages/bad_built_missing_input/docs/README.md b/test/built_packages/bad_built_missing_input/docs/README.md new file mode 100644 index 000000000..952c3c1bb --- /dev/null +++ b/test/built_packages/bad_built_missing_input/docs/README.md @@ -0,0 +1,6 @@ +# Bad Built Package - Stream Missing `input:` Field + +This fixture is intentionally invalid for build-mode validation tests (issue `#549`). +It models a built package where a data stream stream entry does not have the required +`input:` field materialized. Build mode rejects streams that are missing `input:`, +as they represent an incompletely materialised composable package. diff --git a/test/built_packages/bad_built_missing_input/manifest.yml b/test/built_packages/bad_built_missing_input/manifest.yml new file mode 100644 index 000000000..a598b06c2 --- /dev/null +++ b/test/built_packages/bad_built_missing_input/manifest.yml @@ -0,0 +1,23 @@ +format_version: 3.6.0 +name: bad_built_missing_input +title: Bad Built Package - Missing Input +description: Built-package fixture with a stream missing required input field - invalid in build mode (issue #549). +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0 || ^9.0.0' +policy_templates: + - name: events + title: Events logs + description: Collect events data + inputs: + - type: logfile + title: Collect events logs + description: Collecting events log data + multi: false +owner: + github: elastic/foobar + type: community diff --git a/test/built_packages/bad_built_policy_template_package/LICENSE.txt b/test/built_packages/bad_built_policy_template_package/LICENSE.txt new file mode 100644 index 000000000..f6f788a89 --- /dev/null +++ b/test/built_packages/bad_built_policy_template_package/LICENSE.txt @@ -0,0 +1,3 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license diff --git a/test/built_packages/bad_built_policy_template_package/changelog.yml b/test/built_packages/bad_built_policy_template_package/changelog.yml new file mode 100644 index 000000000..e00f88133 --- /dev/null +++ b/test/built_packages/bad_built_policy_template_package/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 diff --git a/test/built_packages/bad_built_policy_template_package/data_stream/events/agent/stream/stream.yml.hbs b/test/built_packages/bad_built_policy_template_package/data_stream/events/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..9390bc05c --- /dev/null +++ b/test/built_packages/bad_built_policy_template_package/data_stream/events/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} diff --git a/test/built_packages/bad_built_policy_template_package/data_stream/events/fields/base-fields.yml b/test/built_packages/bad_built_policy_template_package/data_stream/events/fields/base-fields.yml new file mode 100644 index 000000000..0d1791ffe --- /dev/null +++ b/test/built_packages/bad_built_policy_template_package/data_stream/events/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/built_packages/bad_built_policy_template_package/data_stream/events/fields/fields.yml b/test/built_packages/bad_built_policy_template_package/data_stream/events/fields/fields.yml new file mode 100644 index 000000000..f21b45be3 --- /dev/null +++ b/test/built_packages/bad_built_policy_template_package/data_stream/events/fields/fields.yml @@ -0,0 +1,3 @@ +- name: message + type: keyword + description: Log message. diff --git a/test/built_packages/bad_built_policy_template_package/data_stream/events/manifest.yml b/test/built_packages/bad_built_policy_template_package/data_stream/events/manifest.yml new file mode 100644 index 000000000..d51b2bd1c --- /dev/null +++ b/test/built_packages/bad_built_policy_template_package/data_stream/events/manifest.yml @@ -0,0 +1,15 @@ +title: Events +type: logs +streams: + - input: logfile + title: Events logs + description: Collect events log data + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/*.log diff --git a/test/built_packages/bad_built_policy_template_package/docs/README.md b/test/built_packages/bad_built_policy_template_package/docs/README.md new file mode 100644 index 000000000..bedaff3a1 --- /dev/null +++ b/test/built_packages/bad_built_policy_template_package/docs/README.md @@ -0,0 +1,5 @@ +# Bad Built Package - Policy Template Uses `package:` + +This fixture is intentionally invalid for build-mode validation tests (issue `#549`). +It models a built package where policy template inputs use `package:` instead of +the required materialized `type:` field. diff --git a/test/built_packages/bad_built_policy_template_package/manifest.yml b/test/built_packages/bad_built_policy_template_package/manifest.yml new file mode 100644 index 000000000..c81e0e6e8 --- /dev/null +++ b/test/built_packages/bad_built_policy_template_package/manifest.yml @@ -0,0 +1,22 @@ +format_version: 3.6.0 +name: bad_built_policy_template_package +title: Bad Built Package - Policy Template Package Reference +description: Built-package fixture with a source-only package reference in policy_templates - invalid in build mode (issue #549). +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0 || ^9.0.0' +policy_templates: + - name: events + title: Events logs + description: Collect events data + inputs: + - package: filelog_otel + title: Collect events logs + description: Collecting events log data (source-only package reference — invalid in built packages) +owner: + github: elastic/foobar + type: community diff --git a/test/built_packages/bad_built_stream_package/LICENSE.txt b/test/built_packages/bad_built_stream_package/LICENSE.txt new file mode 100644 index 000000000..f6f788a89 --- /dev/null +++ b/test/built_packages/bad_built_stream_package/LICENSE.txt @@ -0,0 +1,3 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license diff --git a/test/built_packages/bad_built_stream_package/changelog.yml b/test/built_packages/bad_built_stream_package/changelog.yml new file mode 100644 index 000000000..e00f88133 --- /dev/null +++ b/test/built_packages/bad_built_stream_package/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 diff --git a/test/built_packages/bad_built_stream_package/data_stream/events/agent/stream/stream.yml.hbs b/test/built_packages/bad_built_stream_package/data_stream/events/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..9390bc05c --- /dev/null +++ b/test/built_packages/bad_built_stream_package/data_stream/events/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} diff --git a/test/built_packages/bad_built_stream_package/data_stream/events/fields/base-fields.yml b/test/built_packages/bad_built_stream_package/data_stream/events/fields/base-fields.yml new file mode 100644 index 000000000..0d1791ffe --- /dev/null +++ b/test/built_packages/bad_built_stream_package/data_stream/events/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/built_packages/bad_built_stream_package/data_stream/events/fields/fields.yml b/test/built_packages/bad_built_stream_package/data_stream/events/fields/fields.yml new file mode 100644 index 000000000..fa9de06d4 --- /dev/null +++ b/test/built_packages/bad_built_stream_package/data_stream/events/fields/fields.yml @@ -0,0 +1,5 @@ +- name: message + type: keyword + description: Log message. +- name: host.name + type: keyword diff --git a/test/built_packages/bad_built_stream_package/data_stream/events/manifest.yml b/test/built_packages/bad_built_stream_package/data_stream/events/manifest.yml new file mode 100644 index 000000000..16f358447 --- /dev/null +++ b/test/built_packages/bad_built_stream_package/data_stream/events/manifest.yml @@ -0,0 +1,6 @@ +title: Events +type: logs +streams: + - package: filelog_otel + title: Events logs + description: Collect events log data (source-only package reference — invalid in built packages) diff --git a/test/built_packages/bad_built_stream_package/docs/README.md b/test/built_packages/bad_built_stream_package/docs/README.md new file mode 100644 index 000000000..957b72908 --- /dev/null +++ b/test/built_packages/bad_built_stream_package/docs/README.md @@ -0,0 +1,5 @@ +# Bad Built Package - Stream Package Reference + +This fixture is intentionally invalid for build-mode validation tests (issue `#549`). +It uses source-only `package:` references in stream definitions, which must be +rejected under `ModeBuild`. diff --git a/test/built_packages/bad_built_stream_package/manifest.yml b/test/built_packages/bad_built_stream_package/manifest.yml new file mode 100644 index 000000000..f04a853ad --- /dev/null +++ b/test/built_packages/bad_built_stream_package/manifest.yml @@ -0,0 +1,23 @@ +format_version: 3.6.0 +name: bad_built_stream_package +title: Bad Built Package - Stream Package Reference +description: Built-package fixture with a source-only package reference in a data stream - invalid in build mode (issue #549). +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0 || ^9.0.0' +policy_templates: + - name: events + title: Events logs + description: Collect events data + inputs: + - type: logfile + title: Collect events logs + description: Collecting events log data + multi: false +owner: + github: elastic/foobar + type: community diff --git a/test/built_packages/good_built/LICENSE.txt b/test/built_packages/good_built/LICENSE.txt new file mode 100644 index 000000000..f6f788a89 --- /dev/null +++ b/test/built_packages/good_built/LICENSE.txt @@ -0,0 +1,3 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license diff --git a/test/built_packages/good_built/changelog.yml b/test/built_packages/good_built/changelog.yml new file mode 100644 index 000000000..e00f88133 --- /dev/null +++ b/test/built_packages/good_built/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 diff --git a/test/built_packages/good_built/data_stream/events/agent/stream/stream.yml.hbs b/test/built_packages/good_built/data_stream/events/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..9390bc05c --- /dev/null +++ b/test/built_packages/good_built/data_stream/events/agent/stream/stream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} diff --git a/test/built_packages/good_built/data_stream/events/fields/base-fields.yml b/test/built_packages/good_built/data_stream/events/fields/base-fields.yml new file mode 100644 index 000000000..0d1791ffe --- /dev/null +++ b/test/built_packages/good_built/data_stream/events/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/built_packages/good_built/data_stream/events/fields/fields.yml b/test/built_packages/good_built/data_stream/events/fields/fields.yml new file mode 100644 index 000000000..f21b45be3 --- /dev/null +++ b/test/built_packages/good_built/data_stream/events/fields/fields.yml @@ -0,0 +1,3 @@ +- name: message + type: keyword + description: Log message. diff --git a/test/built_packages/good_built/data_stream/events/manifest.yml b/test/built_packages/good_built/data_stream/events/manifest.yml new file mode 100644 index 000000000..d51b2bd1c --- /dev/null +++ b/test/built_packages/good_built/data_stream/events/manifest.yml @@ -0,0 +1,15 @@ +title: Events +type: logs +streams: + - input: logfile + title: Events logs + description: Collect events log data + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/*.log diff --git a/test/built_packages/good_built/docs/README.md b/test/built_packages/good_built/docs/README.md new file mode 100644 index 000000000..c0b01c5ec --- /dev/null +++ b/test/built_packages/good_built/docs/README.md @@ -0,0 +1,5 @@ +# Good Built Package + +This is the canonical minimal built-package fixture used for build-mode validation +tests (issue #549). It has no `_dev/` directories, no `.link` files, and no +`external: ecs` field references — making it valid under `ModeBuild`. diff --git a/test/built_packages/good_built/manifest.yml b/test/built_packages/good_built/manifest.yml new file mode 100644 index 000000000..0dc0d4080 --- /dev/null +++ b/test/built_packages/good_built/manifest.yml @@ -0,0 +1,23 @@ +format_version: 3.6.0 +name: good_built +title: Good Built Package +description: Canonical minimal built-package fixture for build-mode validation tests (issue #549). +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0 || ^9.0.0' +policy_templates: + - name: events + title: Events logs + description: Collect events data + inputs: + - type: logfile + title: Collect events logs + description: Collecting events log data + multi: false +owner: + github: elastic/foobar + type: community diff --git a/test/packages/bad_embedded_ecs/changelog.yml b/test/packages/bad_embedded_ecs/changelog.yml new file mode 100644 index 000000000..27e62eecd --- /dev/null +++ b/test/packages/bad_embedded_ecs/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1 diff --git a/test/packages/bad_embedded_ecs/data_stream/logs/agent/stream/stream.yml.hbs b/test/packages/bad_embedded_ecs/data_stream/logs/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5845510de --- /dev/null +++ b/test/packages/bad_embedded_ecs/data_stream/logs/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/bad_embedded_ecs/data_stream/logs/fields/base-fields.yml b/test/packages/bad_embedded_ecs/data_stream/logs/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/bad_embedded_ecs/data_stream/logs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/bad_embedded_ecs/data_stream/logs/manifest.yml b/test/packages/bad_embedded_ecs/data_stream/logs/manifest.yml new file mode 100644 index 000000000..f4a360472 --- /dev/null +++ b/test/packages/bad_embedded_ecs/data_stream/logs/manifest.yml @@ -0,0 +1,31 @@ +# This data stream intentionally contains _embedded_ecs-* dynamic template keys. +# It serves two purposes: +# 1. Schema coverage: verifies the spec's patternProperties "^_embedded_ecs" block +# accepts these keys (exercised by TestLegacyPreservation_FromPath). +# 2. Semantic rejection: verifies ValidateNoEmbeddedEcsInDynamicTemplates rejects +# them in source mode (exercised by TestSourceMode_BadEmbeddedEcs). +title: Logs +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /var/log/*.log +elasticsearch: + index_template: + mappings: + dynamic_templates: + - _embedded_ecs-ip_to_ip: + mapping: + type: ip + match: ip + - _embedded_ecs-port_to_long: + mapping: + type: long + match: port diff --git a/test/packages/bad_embedded_ecs/docs/README.md b/test/packages/bad_embedded_ecs/docs/README.md new file mode 100644 index 000000000..05f822286 --- /dev/null +++ b/test/packages/bad_embedded_ecs/docs/README.md @@ -0,0 +1,7 @@ +# Bad ECS Embedded in Source + +Test fixture: integration package containing `_embedded_ecs` keys in `dynamic_templates`. + +These keys are auto-injected by `elastic-package` at build time when `import_mappings` is +enabled. In source packages they must not appear. This package is valid in legacy mode but +rejected in source mode by `ValidateNoEmbeddedEcsInDynamicTemplates`. diff --git a/test/packages/bad_embedded_ecs/manifest.yml b/test/packages/bad_embedded_ecs/manifest.yml new file mode 100644 index 000000000..178c7c751 --- /dev/null +++ b/test/packages/bad_embedded_ecs/manifest.yml @@ -0,0 +1,24 @@ +format_version: 3.6.0 +name: bad_embedded_ecs +title: Bad ECS embedded in source +description: > + Integration with _embedded_ecs keys in dynamic_templates. + Valid in legacy mode; rejected in source mode by ValidateNoEmbeddedEcsInDynamicTemplates. +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0' +policy_templates: + - name: logs + title: Logs + description: Collect logs + inputs: + - type: logfile + title: Collect logfile logs + description: Collect logs using logfile input +owner: + github: elastic/foobar + type: community diff --git a/test/packages/good_requires/data_stream/logs/manifest.yml b/test/packages/good_requires/data_stream/logs/manifest.yml index 0ab41203c..6c5e2f0c3 100644 --- a/test/packages/good_requires/data_stream/logs/manifest.yml +++ b/test/packages/good_requires/data_stream/logs/manifest.yml @@ -4,3 +4,7 @@ streams: - package: sql_input title: Apache logs via SQL input description: Collect Apache logs using the SQL input package + vars: + - name: hosts + default: localhost:5432 + - default: 30s diff --git a/test/packages/good_v3/data_stream/ecs_import_mappings/manifest.yml b/test/packages/good_v3/data_stream/ecs_import_mappings/manifest.yml index e7dab8a94..38d6f4102 100644 --- a/test/packages/good_v3/data_stream/ecs_import_mappings/manifest.yml +++ b/test/packages/good_v3/data_stream/ecs_import_mappings/manifest.yml @@ -1,4 +1,9 @@ -title: "Includes ECS imported mappings in manifest" +# Previously contained _embedded_ecs-* entries to test JSON schema acceptance of +# patternProperties "^_embedded_ecs". Those entries were moved to +# test/packages/bad_embedded_ecs/data_stream/logs/manifest.yml (issue #549): +# _embedded_ecs keys are build artifacts and must not appear in source packages. +# This data stream now exercises regular dynamic_templates entries only. +title: "Includes custom dynamic mappings in manifest" type: logs streams: - input: logfile @@ -15,462 +20,11 @@ elasticsearch: index_template: mappings: dynamic_templates: - - _embedded_ecs-ecs_timestamp: - mapping: - ignore_malformed: false - type: date - path_match: '@timestamp' - - _embedded_ecs-data_stream_to_constant: - mapping: - type: constant_keyword - path_match: data_stream.* - - _embedded_ecs-resolved_ip_to_ip: - mapping: - type: ip - match: resolved_ip - - _embedded_ecs-forwarded_ip_to_ip: - mapping: - type: ip - match: forwarded_ip - match_mapping_type: string - - _embedded_ecs-ip_to_ip: - mapping: - type: ip - match: ip + - string_as_keyword: match_mapping_type: string - - _embedded_ecs-port_to_long: - mapping: - type: long - match: port - - _embedded_ecs-thread_id_to_long: - mapping: - type: long - path_match: '*.thread.id' - - _embedded_ecs-status_code_to_long: - mapping: - type: long - match: status_code - - _embedded_ecs-line_to_long: - mapping: - type: long - path_match: '*.file.line' - - _embedded_ecs-priority_to_long: - mapping: - type: long - path_match: log.syslog.priority - - _embedded_ecs-code_to_long: - mapping: - type: long - path_match: '*.facility.code' - - _embedded_ecs-code_to_long: - mapping: - type: long - path_match: '*.severity.code' - - _embedded_ecs-bytes_to_long: - mapping: - type: long - match: bytes - path_unmatch: '*.data.bytes' - - _embedded_ecs-packets_to_long: - mapping: - type: long - match: packets - - _embedded_ecs-public_key_exponent_to_long: - mapping: - type: long - match: public_key_exponent - - _embedded_ecs-severity_to_long: - mapping: - type: long - path_match: event.severity - - _embedded_ecs-duration_to_long: - mapping: - type: long - path_match: event.duration - - _embedded_ecs-pid_to_long: - mapping: - type: long - match: pid - - _embedded_ecs-uptime_to_long: - mapping: - type: long - match: uptime - - _embedded_ecs-sequence_to_long: - mapping: - type: long - match: sequence - - _embedded_ecs-entropy_to_long: - mapping: - type: long - match: '*entropy' - - _embedded_ecs-size_to_long: - mapping: - type: long - match: '*size' - - _embedded_ecs-entrypoint_to_long: - mapping: - type: long - match: entrypoint - - _embedded_ecs-ttl_to_long: - mapping: - type: long - match: ttl - - _embedded_ecs-major_to_long: - mapping: - type: long - match: major - - _embedded_ecs-minor_to_long: - mapping: - type: long - match: minor - - _embedded_ecs-as_number_to_long: - mapping: - type: long - path_match: '*.as.number' - - _embedded_ecs-pgid_to_long: - mapping: - type: long - match: pgid - - _embedded_ecs-exit_code_to_long: - mapping: - type: long - match: exit_code - - _embedded_ecs-chi_to_long: - mapping: - type: long - match: chi2 - - _embedded_ecs-args_count_to_long: - mapping: - type: long - match: args_count - - _embedded_ecs-virtual_address_to_long: - mapping: - type: long - match: virtual_address - - _embedded_ecs-io_text_to_wildcard: - mapping: - type: wildcard - path_match: '*.io.text' - - _embedded_ecs-strings_to_wildcard: - mapping: - type: wildcard - path_match: registry.data.strings - - _embedded_ecs-path_to_wildcard: - mapping: - type: wildcard - path_match: '*url.path' - - _embedded_ecs-message_id_to_wildcard: - mapping: - type: wildcard - match: message_id - - _embedded_ecs-command_line_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: wildcard - match: command_line - - _embedded_ecs-error_stack_trace_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: wildcard - match: stack_trace - - _embedded_ecs-http_content_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: wildcard - path_match: '*.body.content' - - _embedded_ecs-url_full_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: wildcard - path_match: '*.url.full' - - _embedded_ecs-url_original_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: wildcard - path_match: '*.url.original' - - _embedded_ecs-user_agent_original_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: wildcard - path_match: user_agent.original - - _embedded_ecs-error_message_to_match_only: - mapping: - type: match_only_text - path_match: error.message - - _embedded_ecs-message_match_only_text: - mapping: - type: match_only_text - path_match: message - - _embedded_ecs-agent_name_to_keyword: - mapping: - type: keyword - path_match: agent.name - - _embedded_ecs-event_original_non_indexed_keyword: mapping: type: keyword - index: false - doc_values: false - path_match: 'event.original' - - _embedded_ecs-x509_public_key_exponent_non_indexed_keyword: + - long_as_long: + match_mapping_type: long mapping: - type: keyword - index: false - doc_values: false - path_match: '*.x509.public_key_exponent' - - _embedded_ecs-service_name_to_keyword: - mapping: - type: keyword - path_match: '*.service.name' - - _embedded_ecs-sections_name_to_keyword: - mapping: - type: keyword - path_match: '*.sections.name' - - _embedded_ecs-resource_name_to_keyword: - mapping: - type: keyword - path_match: '*.resource.name' - - _embedded_ecs-observer_name_to_keyword: - mapping: - type: keyword - path_match: observer.name - - _embedded_ecs-question_name_to_keyword: - mapping: - type: keyword - path_match: '*.question.name' - - _embedded_ecs-group_name_to_keyword: - mapping: - type: keyword - path_match: '*.group.name' - - _embedded_ecs-geo_name_to_keyword: - mapping: - type: keyword - path_match: '*.geo.name' - - _embedded_ecs-host_name_to_keyword: - mapping: - type: keyword - path_match: host.name - - _embedded_ecs-severity_name_to_keyword: - mapping: - type: keyword - path_match: '*.severity.name' - - _embedded_ecs-title_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: keyword - match: title - - _embedded_ecs-executable_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: keyword - match: executable - - _embedded_ecs-file_path_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: keyword - path_match: '*.file.path' - - _embedded_ecs-file_target_path_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: keyword - path_match: '*.file.target_path' - - _embedded_ecs-name_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: keyword - match: name - - _embedded_ecs-full_name_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: keyword - match: full_name - - _embedded_ecs-os_full_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: keyword - path_match: '*.os.full' - - _embedded_ecs-working_directory_to_multifield: - mapping: - fields: - text: - type: match_only_text - type: keyword - match: working_directory - - _embedded_ecs-timestamp_to_date: - mapping: - type: date - match: timestamp - - _embedded_ecs-delivery_timestamp_to_date: - mapping: - type: date - match: delivery_timestamp - - _embedded_ecs-not_after_to_date: - mapping: - type: date - match: not_after - - _embedded_ecs-not_before_to_date: - mapping: - type: date - match: not_before - - _embedded_ecs-accessed_to_date: - mapping: - type: date - match: accessed - - _embedded_ecs-origination_timestamp_to_date: - mapping: - type: date - match: origination_timestamp - - _embedded_ecs-created_to_date: - mapping: - type: date - match: created - - _embedded_ecs-installed_to_date: - mapping: - type: date - match: installed - - _embedded_ecs-creation_date_to_date: - mapping: - type: date - match: creation_date - - _embedded_ecs-ctime_to_date: - mapping: - type: date - match: ctime - - _embedded_ecs-mtime_to_date: - mapping: - type: date - match: mtime - - _embedded_ecs-ingested_to_date: - mapping: - type: date - match: ingested - - _embedded_ecs-start_to_date: - mapping: - type: date - match: start - - _embedded_ecs-end_to_date: - mapping: - type: date - match: end - - _embedded_ecs-score_base_to_float: - mapping: - type: float - path_match: '*.score.base' - - _embedded_ecs-score_temporal_to_float: - mapping: - type: float - path_match: '*.score.temporal' - - _embedded_ecs-score_to_float: - mapping: - type: float - match: '*_score' - - _embedded_ecs-score_norm_to_float: - mapping: - type: float - match: '*_score_norm' - - _embedded_ecs-usage_to_float: - mapping: - scaling_factor: 1000 - type: scaled_float - match: usage - - _embedded_ecs-location_to_geo_point: - mapping: - type: geo_point - match: location - - _embedded_ecs-same_as_process_to_boolean: - mapping: - type: boolean - match: same_as_process - - _embedded_ecs-established_to_boolean: - mapping: - type: boolean - match: established - - _embedded_ecs-resumed_to_boolean: - mapping: - type: boolean - match: resumed - - _embedded_ecs-max_bytes_per_process_exceeded_to_boolean: - mapping: - type: boolean - match: max_bytes_per_process_exceeded - - _embedded_ecs-interactive_to_boolean: - mapping: - type: boolean - match: interactive - - _embedded_ecs-exists_to_boolean: - mapping: - type: boolean - match: exists - - _embedded_ecs-trusted_to_boolean: - mapping: - type: boolean - match: trusted - - _embedded_ecs-valid_to_boolean: - mapping: - type: boolean - match: valid - - _embedded_ecs-go_stripped_to_boolean: - mapping: - type: boolean - match: go_stripped - - _embedded_ecs-coldstart_to_boolean: - mapping: - type: boolean - match: coldstart - - _embedded_ecs-exports_to_flattened: - mapping: - type: flattened - match: exports - - _embedded_ecs-structured_data_to_flattened: - mapping: - type: flattened - match: structured_data - - _embedded_ecs-imports_to_flattened: - mapping: - type: flattened - match: '*imports' - - _embedded_ecs-attachments_to_nested: - mapping: - type: nested - match: attachments - - _embedded_ecs-segments_to_nested: - mapping: - type: nested - match: segments - - _embedded_ecs-elf_sections_to_nested: - mapping: - type: nested - path_match: '*.elf.sections' - - _embedded_ecs-pe_sections_to_nested: - mapping: - type: nested - path_match: '*.pe.sections' - - _embedded_ecs-macho_sections_to_nested: - mapping: - type: nested - path_match: '*.macho.sections' + type: long