From b57aaedd3bb8aaceb7a482239e83f068ca74c1f3 Mon Sep 17 00:00:00 2001 From: jdkurma Date: Sun, 22 Feb 2026 20:59:55 -0600 Subject: [PATCH 01/13] Add semantic validator for policy template datastream categories --- ...e_policy_template_datastream_categories.go | 147 ++++++++++++ ...icy_template_datastream_categories_test.go | 213 ++++++++++++++++++ code/go/internal/validator/spec.go | 1 + code/go/pkg/validator/validator_test.go | 7 + spec/changelog.yml | 3 + .../changelog.yml | 5 + .../mylogs/agent/stream/stream.yml.hbs | 1 + .../data_stream/mylogs/fields/fields.yml | 12 + .../data_stream/mylogs/manifest.yml | 8 + .../docs/README.md | 1 + .../manifest.yml | 26 +++ .../changelog.yml | 5 + .../mylogs/agent/stream/stream.yml.hbs | 1 + .../data_stream/mylogs/fields/fields.yml | 12 + .../data_stream/mylogs/manifest.yml | 8 + .../docs/README.md | 1 + .../manifest.yml | 26 +++ 17 files changed, 477 insertions(+) create mode 100644 code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go create mode 100644 code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go create mode 100644 test/packages/bad_datastream_categories_mismatch/changelog.yml create mode 100644 test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/agent/stream/stream.yml.hbs create mode 100644 test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/fields/fields.yml create mode 100644 test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/manifest.yml create mode 100644 test/packages/bad_datastream_categories_mismatch/docs/README.md create mode 100644 test/packages/bad_datastream_categories_mismatch/manifest.yml create mode 100644 test/packages/good_datastream_categories_match/changelog.yml create mode 100644 test/packages/good_datastream_categories_match/data_stream/mylogs/agent/stream/stream.yml.hbs create mode 100644 test/packages/good_datastream_categories_match/data_stream/mylogs/fields/fields.yml create mode 100644 test/packages/good_datastream_categories_match/data_stream/mylogs/manifest.yml create mode 100644 test/packages/good_datastream_categories_match/docs/README.md create mode 100644 test/packages/good_datastream_categories_match/manifest.yml diff --git a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go new file mode 100644 index 000000000..b9a90d599 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go @@ -0,0 +1,147 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "io/fs" + "path" + "sort" + + "gopkg.in/yaml.v3" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" +) + +type policyTemplateWithCategories struct { + Name string `yaml:"name"` + DataStreams []string `yaml:"data_streams"` + Categories []string `yaml:"categories"` +} + +type packageManifestWithCategories struct { + Type string `yaml:"type"` + PolicyTemplates []policyTemplateWithCategories `yaml:"policy_templates"` +} + +type dataStreamManifestWithCategories struct { + Categories []string `yaml:"categories"` +} + +// ValidatePolicyTemplateDatastreamCategories validates that when a policy template +// entry in the package manifest.yml defines categories, those categories match the +// categories defined in the manifest.yml of each referenced data stream. +// Data stream manifests without a categories field are skipped. +func ValidatePolicyTemplateDatastreamCategories(fsys fspath.FS) specerrors.ValidationErrors { + var errs specerrors.ValidationErrors + + manifestPath := "manifest.yml" + data, err := fs.ReadFile(fsys, manifestPath) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToReadManifest)} + } + + var manifest packageManifestWithCategories + if err := yaml.Unmarshal(data, &manifest); err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)} + } + + // only validate integration type packages + if manifest.Type != packageTypeIntegration { + return nil + } + + dsCategories, err := readDataStreamManifestCategories(fsys) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} + } + + for _, pt := range manifest.PolicyTemplates { + // skip policy templates that don't define both categories and data_streams + if len(pt.Categories) == 0 || len(pt.DataStreams) == 0 { + continue + } + + for _, dsName := range pt.DataStreams { + dsCats, hasCats := dsCategories[dsName] + if !hasCats { + // data stream manifest has no categories field — nothing to validate + continue + } + + if !categoriesEqual(pt.Categories, dsCats) { + dsManifestPath := path.Join("data_stream", dsName, "manifest.yml") + errs = append(errs, specerrors.NewStructuredErrorf( + "file \"%s\" is invalid: policy template \"%s\" categories %v do not match data stream \"%s\" manifest categories %v (defined in \"%s\")", + fsys.Path(manifestPath), + pt.Name, + pt.Categories, + dsName, + dsCats, + fsys.Path(dsManifestPath), + )) + } + } + } + + return errs +} + +// readDataStreamManifestCategories reads the categories field from every +// data_stream/*/manifest.yml and returns a map of data stream name to its categories. +// Data streams without a categories field are omitted from the map. +func readDataStreamManifestCategories(fsys fspath.FS) (map[string][]string, error) { + result := make(map[string][]string) + + manifests, err := fs.Glob(fsys, "data_stream/*/manifest.yml") + if err != nil { + return nil, err + } + + for _, file := range manifests { + data, err := fs.ReadFile(fsys, file) + if err != nil { + return nil, err + } + + var m dataStreamManifestWithCategories + if err := yaml.Unmarshal(data, &m); err != nil { + return nil, err + } + + if len(m.Categories) == 0 { + continue + } + + // path is data_stream/{name}/manifest.yml — extract the name component + dsName := path.Base(path.Dir(file)) + result[dsName] = m.Categories + } + + return result, nil +} + +// categoriesEqual returns true if both slices contain exactly the same set of +// categories, regardless of order. +func categoriesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + aCopy := make([]string, len(a)) + bCopy := make([]string, len(b)) + copy(aCopy, a) + copy(bCopy, b) + sort.Strings(aCopy) + sort.Strings(bCopy) + for i := range aCopy { + if aCopy[i] != bCopy[i] { + return false + } + } + return true +} diff --git a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go new file mode 100644 index 000000000..5988f1656 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go @@ -0,0 +1,213 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" +) + +func writeManifest(t *testing.T, dir, content string) { + t.Helper() + err := os.WriteFile(filepath.Join(dir, "manifest.yml"), []byte(content), 0o644) + require.NoError(t, err) +} + +func writeDataStreamManifest(t *testing.T, dir, dsName, content string) { + t.Helper() + dsDir := filepath.Join(dir, "data_stream", dsName) + err := os.MkdirAll(dsDir, 0o755) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(dsDir, "manifest.yml"), []byte(content), 0o644) + require.NoError(t, err) +} + +func TestValidatePolicyTemplateDatastreamCategories_Match(t *testing.T) { + d := t.TempDir() + + writeManifest(t, d, ` +type: integration +policy_templates: + - name: mytemplate + data_streams: + - mylogs + categories: + - observability + - network +`) + writeDataStreamManifest(t, d, "mylogs", ` +title: My Logs +categories: + - network + - observability +`) + + errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) + assert.Empty(t, errs) +} + +func TestValidatePolicyTemplateDatastreamCategories_Mismatch(t *testing.T) { + d := t.TempDir() + + writeManifest(t, d, ` +type: integration +policy_templates: + - name: mytemplate + data_streams: + - mylogs + categories: + - observability +`) + writeDataStreamManifest(t, d, "mylogs", ` +title: My Logs +categories: + - observability + - security +`) + + errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) + assert.Len(t, errs, 1) + assert.Contains(t, errs[0].Error(), `policy template "mytemplate"`) + assert.Contains(t, errs[0].Error(), `"mylogs"`) +} + +func TestValidatePolicyTemplateDatastreamCategories_NoDataStreamCategories(t *testing.T) { + // Data stream manifest without categories field → should pass (nothing to validate) + d := t.TempDir() + + writeManifest(t, d, ` +type: integration +policy_templates: + - name: mytemplate + data_streams: + - mylogs + categories: + - observability +`) + writeDataStreamManifest(t, d, "mylogs", ` +title: My Logs +type: logs +streams: + - input: logfile +`) + + errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) + assert.Empty(t, errs) +} + +func TestValidatePolicyTemplateDatastreamCategories_NoPolicyTemplateCategories(t *testing.T) { + // Policy template without categories field → should pass (nothing to validate) + d := t.TempDir() + + writeManifest(t, d, ` +type: integration +policy_templates: + - name: mytemplate + data_streams: + - mylogs +`) + writeDataStreamManifest(t, d, "mylogs", ` +title: My Logs +categories: + - observability +`) + + errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) + assert.Empty(t, errs) +} + +func TestValidatePolicyTemplateDatastreamCategories_MultipleDataStreams(t *testing.T) { + // Multiple data streams — one matches, one doesn't + d := t.TempDir() + + writeManifest(t, d, ` +type: integration +policy_templates: + - name: mytemplate + data_streams: + - logs_ok + - logs_bad + categories: + - observability +`) + writeDataStreamManifest(t, d, "logs_ok", ` +title: OK Logs +categories: + - observability +`) + writeDataStreamManifest(t, d, "logs_bad", ` +title: Bad Logs +categories: + - security +`) + + errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) + assert.Len(t, errs, 1) + assert.Contains(t, errs[0].Error(), `"logs_bad"`) +} + +func TestValidatePolicyTemplateDatastreamCategories_MultiplePolicyTemplates(t *testing.T) { + // Two policy templates — each has one mismatch + d := t.TempDir() + + writeManifest(t, d, ` +type: integration +policy_templates: + - name: template_a + data_streams: + - ds_a + categories: + - observability + - name: template_b + data_streams: + - ds_b + categories: + - network +`) + writeDataStreamManifest(t, d, "ds_a", ` +title: DS A +categories: + - security +`) + writeDataStreamManifest(t, d, "ds_b", ` +title: DS B +categories: + - network +`) + + errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) + assert.Len(t, errs, 1) + assert.Contains(t, errs[0].Error(), `"template_a"`) + assert.Contains(t, errs[0].Error(), `"ds_a"`) +} + +func TestValidatePolicyTemplateDatastreamCategories_NonIntegrationPackage(t *testing.T) { + // Non-integration packages are skipped + d := t.TempDir() + + writeManifest(t, d, ` +type: input +policy_templates: + - name: mytemplate + data_streams: + - mylogs + categories: + - observability +`) + writeDataStreamManifest(t, d, "mylogs", ` +title: My Logs +categories: + - security +`) + + errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) + assert.Empty(t, errs) +} diff --git a/code/go/internal/validator/spec.go b/code/go/internal/validator/spec.go index aa465cf39..b283bfdd5 100644 --- a/code/go/internal/validator/spec.go +++ b/code/go/internal/validator/spec.go @@ -226,6 +226,7 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules {fn: semantic.ValidateInputDynamicSignalTypes}, {fn: semantic.ValidateMinimumAgentVersion}, {fn: semantic.ValidateIntegrationPolicyTemplates, types: []string{"integration"}}, + {fn: semantic.ValidatePolicyTemplateDatastreamCategories, types: []string{"integration"}}, {fn: semantic.ValidatePipelineTags, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, {fn: semantic.ValidateStaticHandlebarsFiles, types: []string{"integration", "input"}}, {fn: semantic.ValidateKibanaTagDuplicates}, diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index d9af3d782..a37ece20f 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -45,6 +45,7 @@ func TestValidateFile(t *testing.T) { "good_alert_rule_templates": {}, "good_requires": {}, "good_package_reference_policy_template": {}, + "good_datastream_categories_match": {}, "deploy_custom_agent": {}, "deploy_custom_agent_multi_services": {}, "deploy_docker": {}, @@ -141,6 +142,12 @@ func TestValidateFile(t *testing.T) { "field (root): Additional property release is not allowed", }, }, + "bad_datastream_categories_mismatch": { + "manifest.yml", + []string{ + fmt.Sprintf(`policy template "mytemplate" categories [observability] do not match data stream "mylogs" manifest categories [security] (defined in "%sbad_datastream_categories_mismatch/data_stream/mylogs/manifest.yml")`, osTestBasePath), + }, + }, "bad_custom_ilm_policy": { "data_stream/test/manifest.yml", []string{ diff --git a/spec/changelog.yml b/spec/changelog.yml index a00919beb..cbe464d27 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -41,6 +41,9 @@ - description: Add support for package dependencies in integration packages via requires field with input and content package types. type: enhancement link: https://github.com/elastic/package-spec/pull/1071 + - description: Add semantic validator to verify data stream manifest categories match policy template categories. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1095 - version: 3.5.7 changes: - description: Allow _dev directory for content-only packages; use _dev/shared for development files (e.g. dashboard YML sources). diff --git a/test/packages/bad_datastream_categories_mismatch/changelog.yml b/test/packages/bad_datastream_categories_mismatch/changelog.yml new file mode 100644 index 000000000..27e62eecd --- /dev/null +++ b/test/packages/bad_datastream_categories_mismatch/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1 diff --git a/test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/agent/stream/stream.yml.hbs b/test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..767dd8f8a --- /dev/null +++ b/test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/agent/stream/stream.yml.hbs @@ -0,0 +1 @@ +{{fields "stream"}} \ No newline at end of file diff --git a/test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/fields/fields.yml b/test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/manifest.yml b/test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/manifest.yml new file mode 100644 index 000000000..9b8b61a92 --- /dev/null +++ b/test/packages/bad_datastream_categories_mismatch/data_stream/mylogs/manifest.yml @@ -0,0 +1,8 @@ +title: My logs +type: logs +categories: + - security +streams: + - input: logfile + title: My logs + description: Collect my logs diff --git a/test/packages/bad_datastream_categories_mismatch/docs/README.md b/test/packages/bad_datastream_categories_mismatch/docs/README.md new file mode 100644 index 000000000..5f19b37d5 --- /dev/null +++ b/test/packages/bad_datastream_categories_mismatch/docs/README.md @@ -0,0 +1 @@ +# Test package \ No newline at end of file diff --git a/test/packages/bad_datastream_categories_mismatch/manifest.yml b/test/packages/bad_datastream_categories_mismatch/manifest.yml new file mode 100644 index 000000000..74c120959 --- /dev/null +++ b/test/packages/bad_datastream_categories_mismatch/manifest.yml @@ -0,0 +1,26 @@ +format_version: 3.6.0 +name: bad_datastream_categories_mismatch +title: Bad package with mismatched datastream categories +description: Policy template categories do not match the data stream manifest categories. +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0' +policy_templates: + - name: mytemplate + title: My template + description: Collect my logs + data_streams: + - mylogs + categories: + - observability + inputs: + - type: logfile + title: Collect my logs + description: Collecting logs via log input +owner: + github: elastic/foobar + type: community diff --git a/test/packages/good_datastream_categories_match/changelog.yml b/test/packages/good_datastream_categories_match/changelog.yml new file mode 100644 index 000000000..27e62eecd --- /dev/null +++ b/test/packages/good_datastream_categories_match/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1 diff --git a/test/packages/good_datastream_categories_match/data_stream/mylogs/agent/stream/stream.yml.hbs b/test/packages/good_datastream_categories_match/data_stream/mylogs/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..767dd8f8a --- /dev/null +++ b/test/packages/good_datastream_categories_match/data_stream/mylogs/agent/stream/stream.yml.hbs @@ -0,0 +1 @@ +{{fields "stream"}} \ No newline at end of file diff --git a/test/packages/good_datastream_categories_match/data_stream/mylogs/fields/fields.yml b/test/packages/good_datastream_categories_match/data_stream/mylogs/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_datastream_categories_match/data_stream/mylogs/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_datastream_categories_match/data_stream/mylogs/manifest.yml b/test/packages/good_datastream_categories_match/data_stream/mylogs/manifest.yml new file mode 100644 index 000000000..7c9fc4396 --- /dev/null +++ b/test/packages/good_datastream_categories_match/data_stream/mylogs/manifest.yml @@ -0,0 +1,8 @@ +title: My logs +type: logs +categories: + - observability +streams: + - input: logfile + title: My logs + description: Collect my logs diff --git a/test/packages/good_datastream_categories_match/docs/README.md b/test/packages/good_datastream_categories_match/docs/README.md new file mode 100644 index 000000000..5f19b37d5 --- /dev/null +++ b/test/packages/good_datastream_categories_match/docs/README.md @@ -0,0 +1 @@ +# Test package \ No newline at end of file diff --git a/test/packages/good_datastream_categories_match/manifest.yml b/test/packages/good_datastream_categories_match/manifest.yml new file mode 100644 index 000000000..7284cfcd5 --- /dev/null +++ b/test/packages/good_datastream_categories_match/manifest.yml @@ -0,0 +1,26 @@ +format_version: 3.6.0 +name: good_datastream_categories_match +title: Good package with matching datastream categories +description: Policy template categories match the data stream manifest categories. +version: 0.0.1 +type: integration +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0' +policy_templates: + - name: mytemplate + title: My template + description: Collect my logs + data_streams: + - mylogs + categories: + - observability + inputs: + - type: logfile + title: Collect my logs + description: Collecting logs via log input +owner: + github: elastic/foobar + type: community From 3c49b8dce4a1fe66b559bf62632030d3bc9f013e Mon Sep 17 00:00:00 2001 From: jdkurma Date: Wed, 4 Mar 2026 18:25:46 -0600 Subject: [PATCH 02/13] add validation between package and datastream categories --- .../validate_datastream_package_categories.go | 126 +++++++++++ ...date_datastream_package_categories_test.go | 197 ++++++++++++++++++ ...icy_template_datastream_categories_test.go | 159 +++++++------- code/go/internal/validator/spec.go | 1 + code/go/pkg/validator/validator_test.go | 7 + .../changelog.yml | 5 + .../mylogs/agent/stream/stream.yml.hbs | 1 + .../data_stream/mylogs/fields/fields.yml | 12 ++ .../data_stream/mylogs/manifest.yml | 8 + .../manifest.yml | 26 +++ .../changelog.yml | 5 + .../mylogs/agent/stream/stream.yml.hbs | 1 + .../data_stream/mylogs/fields/fields.yml | 12 ++ .../data_stream/mylogs/manifest.yml | 8 + .../manifest.yml | 26 +++ 15 files changed, 514 insertions(+), 80 deletions(-) create mode 100644 code/go/internal/validator/semantic/validate_datastream_package_categories.go create mode 100644 code/go/internal/validator/semantic/validate_datastream_package_categories_test.go create mode 100644 test/packages/bad_datastream_package_categories/changelog.yml create mode 100644 test/packages/bad_datastream_package_categories/data_stream/mylogs/agent/stream/stream.yml.hbs create mode 100644 test/packages/bad_datastream_package_categories/data_stream/mylogs/fields/fields.yml create mode 100644 test/packages/bad_datastream_package_categories/data_stream/mylogs/manifest.yml create mode 100644 test/packages/bad_datastream_package_categories/manifest.yml create mode 100644 test/packages/good_datastream_package_categories/changelog.yml create mode 100644 test/packages/good_datastream_package_categories/data_stream/mylogs/agent/stream/stream.yml.hbs create mode 100644 test/packages/good_datastream_package_categories/data_stream/mylogs/fields/fields.yml create mode 100644 test/packages/good_datastream_package_categories/data_stream/mylogs/manifest.yml create mode 100644 test/packages/good_datastream_package_categories/manifest.yml diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories.go b/code/go/internal/validator/semantic/validate_datastream_package_categories.go new file mode 100644 index 000000000..0032ae4c2 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories.go @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "fmt" + "io" + "io/fs" + "net/http" + "path" + "slices" + "sort" + "time" + + "gopkg.in/yaml.v3" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" +) + +const packageRegistryCategoriesURL = "https://raw.githubusercontent.com/elastic/package-registry/main/categories/categories.yml" + +type registryCategories struct { + Categories map[string]struct { + Title string `yaml:"title"` + Subcategories map[string]struct { + Title string `yaml:"title"` + } `yaml:"subcategories"` + } `yaml:"categories"` +} + +func fetchRegistryParentCategories() (map[string]struct{}, error) { + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Get(packageRegistryCategoriesURL) + if err != nil { + return nil, fmt.Errorf("failed to fetch categories from package registry: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read categories response: %w", err) + } + + var rc registryCategories + if err := yaml.Unmarshal(body, &rc); err != nil { + return nil, fmt.Errorf("failed to parse categories YAML: %w", err) + } + + parentCategories := make(map[string]struct{}, len(rc.Categories)) + for id := range rc.Categories { + parentCategories[id] = struct{}{} + } + return parentCategories, nil +} + +type packageManifestWithPackageCategories struct { + Type string `yaml:"type"` + Categories []string `yaml:"categories"` +} + +// ValidateDatastreamPackageCategories validates that the package manifest +// categories include all parent-level categories present in any data stream +// manifest. Parent categories are determined by fetching the package registry +// categories.yml. Data stream manifests without a categories field are skipped. +func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationErrors { + manifestPath := "manifest.yml" + data, err := fs.ReadFile(fsys, manifestPath) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToReadManifest)} + } + + var manifest packageManifestWithPackageCategories + if err := yaml.Unmarshal(data, &manifest); err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)} + } + + if manifest.Type != packageTypeIntegration { + return nil + } + + dsCategories, err := readDataStreamManifestCategories(fsys) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} + } + + if len(dsCategories) == 0 { + return nil + } + + parentCategories, err := fetchRegistryParentCategories() + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to load registry categories: %w", fsys.Path(manifestPath), err)} + } + + var errs specerrors.ValidationErrors + for dsName, dsCats := range dsCategories { + var missingCats []string + for _, dsCat := range dsCats { + if _, isParent := parentCategories[dsCat]; isParent && !slices.Contains(manifest.Categories, dsCat) { + missingCats = append(missingCats, dsCat) + } + } + + if len(missingCats) > 0 { + sort.Strings(missingCats) + dsManifestPath := path.Join("data_stream", dsName, "manifest.yml") + errs = append(errs, specerrors.NewStructuredErrorf( + "file \"%s\" is invalid: package manifest categories %v are missing parent categories %v from data stream \"%s\" (defined in \"%s\")", + fsys.Path(manifestPath), + manifest.Categories, + missingCats, + dsName, + fsys.Path(dsManifestPath), + )) + } + } + + return errs +} diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories_test.go b/code/go/internal/validator/semantic/validate_datastream_package_categories_test.go new file mode 100644 index 000000000..0d3eb8b67 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories_test.go @@ -0,0 +1,197 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" +) + +func TestValidateDatastreamPackageCategories(t *testing.T) { + cases := []struct { + title string + setup func(t *testing.T, dir string) + expectedErrs []string + }{ + { + title: "package includes datastream parent category", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +categories: + - security +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +categories: + - security +type: logs +`) + }, + }, + { + title: "package missing datastream parent category", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +categories: + - observability +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +categories: + - security +type: logs +`) + }, + expectedErrs: []string{ + `package manifest categories [observability] are missing parent categories [security] from data stream "mylogs"`, + }, + }, + { + title: "datastream has subcategory only, no parent to enforce", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +categories: + - observability +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +categories: + - credential_management +type: logs +`) + }, + }, + { + title: "data stream manifest has no categories field, skipped", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +categories: + - observability +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +type: logs +`) + }, + }, + { + title: "package has no categories but datastream does", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +categories: + - security +type: logs +`) + }, + expectedErrs: []string{ + `package manifest categories [] are missing parent categories [security] from data stream "mylogs"`, + }, + }, + { + title: "package has subset, missing one datastream parent category", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +categories: + - security +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +categories: + - security + - observability +type: logs +`) + }, + expectedErrs: []string{`are missing parent categories [observability] from data stream "mylogs"`}, + }, + { + title: "non-integration packages are skipped", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: input +categories: + - observability +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +categories: + - security +type: logs +`) + }, + }, + { + title: "multiple datastreams, one triggers error", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +categories: + - security +`) + writeDataStreamManifest(t, dir, "auditlogs", ` +title: Audit Logs +categories: + - security +type: logs +`) + writeDataStreamManifest(t, dir, "netlogs", ` +title: Net Logs +categories: + - network +type: logs +`) + }, + expectedErrs: []string{`are missing parent categories [network] from data stream "netlogs"`}, + }, + { + title: "datastream does not need all package categories", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +categories: + - security + - observability +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +categories: + - security +type: logs +`) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.title, func(t *testing.T) { + dir := t.TempDir() + tc.setup(t, dir) + + errs := ValidateDatastreamPackageCategories(fspath.DirFS(dir)) + + if len(tc.expectedErrs) == 0 { + assert.Empty(t, errs) + } else { + require.Len(t, errs, 1) + for _, expected := range tc.expectedErrs { + assert.Contains(t, errs[0].Error(), expected) + } + } + }) + } +} diff --git a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go index 5988f1656..b5d069972 100644 --- a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go +++ b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go @@ -30,10 +30,16 @@ func writeDataStreamManifest(t *testing.T, dir, dsName, content string) { require.NoError(t, err) } -func TestValidatePolicyTemplateDatastreamCategories_Match(t *testing.T) { - d := t.TempDir() - - writeManifest(t, d, ` +func TestValidatePolicyTemplateDatastreamCategories(t *testing.T) { + cases := []struct { + title string + setup func(t *testing.T, dir string) + expectedErrs []string + }{ + { + title: "categories match", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` type: integration policy_templates: - name: mytemplate @@ -43,21 +49,18 @@ policy_templates: - observability - network `) - writeDataStreamManifest(t, d, "mylogs", ` + writeDataStreamManifest(t, dir, "mylogs", ` title: My Logs categories: - network - observability `) - - errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) - assert.Empty(t, errs) -} - -func TestValidatePolicyTemplateDatastreamCategories_Mismatch(t *testing.T) { - d := t.TempDir() - - writeManifest(t, d, ` + }, + }, + { + title: "categories mismatch", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` type: integration policy_templates: - name: mytemplate @@ -66,24 +69,19 @@ policy_templates: categories: - observability `) - writeDataStreamManifest(t, d, "mylogs", ` + writeDataStreamManifest(t, dir, "mylogs", ` title: My Logs categories: - observability - security `) - - errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) - assert.Len(t, errs, 1) - assert.Contains(t, errs[0].Error(), `policy template "mytemplate"`) - assert.Contains(t, errs[0].Error(), `"mylogs"`) -} - -func TestValidatePolicyTemplateDatastreamCategories_NoDataStreamCategories(t *testing.T) { - // Data stream manifest without categories field → should pass (nothing to validate) - d := t.TempDir() - - writeManifest(t, d, ` + }, + expectedErrs: []string{`policy template "mytemplate"`, `"mylogs"`}, + }, + { + title: "data stream manifest has no categories field", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` type: integration policy_templates: - name: mytemplate @@ -92,43 +90,35 @@ policy_templates: categories: - observability `) - writeDataStreamManifest(t, d, "mylogs", ` + writeDataStreamManifest(t, dir, "mylogs", ` title: My Logs type: logs streams: - input: logfile `) - - errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) - assert.Empty(t, errs) -} - -func TestValidatePolicyTemplateDatastreamCategories_NoPolicyTemplateCategories(t *testing.T) { - // Policy template without categories field → should pass (nothing to validate) - d := t.TempDir() - - writeManifest(t, d, ` + }, + }, + { + title: "policy template has no categories field", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` type: integration policy_templates: - name: mytemplate data_streams: - mylogs `) - writeDataStreamManifest(t, d, "mylogs", ` + writeDataStreamManifest(t, dir, "mylogs", ` title: My Logs categories: - observability `) - - errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) - assert.Empty(t, errs) -} - -func TestValidatePolicyTemplateDatastreamCategories_MultipleDataStreams(t *testing.T) { - // Multiple data streams — one matches, one doesn't - d := t.TempDir() - - writeManifest(t, d, ` + }, + }, + { + title: "multiple data streams — one matches, one does not", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` type: integration policy_templates: - name: mytemplate @@ -138,27 +128,23 @@ policy_templates: categories: - observability `) - writeDataStreamManifest(t, d, "logs_ok", ` + writeDataStreamManifest(t, dir, "logs_ok", ` title: OK Logs categories: - observability `) - writeDataStreamManifest(t, d, "logs_bad", ` + writeDataStreamManifest(t, dir, "logs_bad", ` title: Bad Logs categories: - security `) - - errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) - assert.Len(t, errs, 1) - assert.Contains(t, errs[0].Error(), `"logs_bad"`) -} - -func TestValidatePolicyTemplateDatastreamCategories_MultiplePolicyTemplates(t *testing.T) { - // Two policy templates — each has one mismatch - d := t.TempDir() - - writeManifest(t, d, ` + }, + expectedErrs: []string{`"logs_bad"`}, + }, + { + title: "multiple policy templates — one mismatch", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` type: integration policy_templates: - name: template_a @@ -172,28 +158,23 @@ policy_templates: categories: - network `) - writeDataStreamManifest(t, d, "ds_a", ` + writeDataStreamManifest(t, dir, "ds_a", ` title: DS A categories: - security `) - writeDataStreamManifest(t, d, "ds_b", ` + writeDataStreamManifest(t, dir, "ds_b", ` title: DS B categories: - network `) - - errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) - assert.Len(t, errs, 1) - assert.Contains(t, errs[0].Error(), `"template_a"`) - assert.Contains(t, errs[0].Error(), `"ds_a"`) -} - -func TestValidatePolicyTemplateDatastreamCategories_NonIntegrationPackage(t *testing.T) { - // Non-integration packages are skipped - d := t.TempDir() - - writeManifest(t, d, ` + }, + expectedErrs: []string{`"template_a"`, `"ds_a"`}, + }, + { + title: "non-integration packages are skipped", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` type: input policy_templates: - name: mytemplate @@ -202,12 +183,30 @@ policy_templates: categories: - observability `) - writeDataStreamManifest(t, d, "mylogs", ` + writeDataStreamManifest(t, dir, "mylogs", ` title: My Logs categories: - security `) - - errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(d)) - assert.Empty(t, errs) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.title, func(t *testing.T) { + dir := t.TempDir() + tc.setup(t, dir) + + errs := ValidatePolicyTemplateDatastreamCategories(fspath.DirFS(dir)) + + if len(tc.expectedErrs) == 0 { + assert.Empty(t, errs) + } else { + require.Len(t, errs, 1) + for _, expected := range tc.expectedErrs { + assert.Contains(t, errs[0].Error(), expected) + } + } + }) + } } diff --git a/code/go/internal/validator/spec.go b/code/go/internal/validator/spec.go index b283bfdd5..f7b1738ab 100644 --- a/code/go/internal/validator/spec.go +++ b/code/go/internal/validator/spec.go @@ -227,6 +227,7 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules {fn: semantic.ValidateMinimumAgentVersion}, {fn: semantic.ValidateIntegrationPolicyTemplates, types: []string{"integration"}}, {fn: semantic.ValidatePolicyTemplateDatastreamCategories, types: []string{"integration"}}, + {fn: semantic.ValidateDatastreamPackageCategories, types: []string{"integration"}}, {fn: semantic.ValidatePipelineTags, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, {fn: semantic.ValidateStaticHandlebarsFiles, types: []string{"integration", "input"}}, {fn: semantic.ValidateKibanaTagDuplicates}, diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index a37ece20f..39d6396e6 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -46,6 +46,7 @@ func TestValidateFile(t *testing.T) { "good_requires": {}, "good_package_reference_policy_template": {}, "good_datastream_categories_match": {}, + "good_datastream_package_categories": {}, "deploy_custom_agent": {}, "deploy_custom_agent_multi_services": {}, "deploy_docker": {}, @@ -148,6 +149,12 @@ func TestValidateFile(t *testing.T) { fmt.Sprintf(`policy template "mytemplate" categories [observability] do not match data stream "mylogs" manifest categories [security] (defined in "%sbad_datastream_categories_mismatch/data_stream/mylogs/manifest.yml")`, osTestBasePath), }, }, + "bad_datastream_package_categories": { + "manifest.yml", + []string{ + fmt.Sprintf(`package manifest categories [observability] are missing parent categories [security] from data stream "mylogs" (defined in "%sbad_datastream_package_categories/data_stream/mylogs/manifest.yml")`, osTestBasePath), + }, + }, "bad_custom_ilm_policy": { "data_stream/test/manifest.yml", []string{ diff --git a/test/packages/bad_datastream_package_categories/changelog.yml b/test/packages/bad_datastream_package_categories/changelog.yml new file mode 100644 index 000000000..27e62eecd --- /dev/null +++ b/test/packages/bad_datastream_package_categories/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1 diff --git a/test/packages/bad_datastream_package_categories/data_stream/mylogs/agent/stream/stream.yml.hbs b/test/packages/bad_datastream_package_categories/data_stream/mylogs/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..ad7430eeb --- /dev/null +++ b/test/packages/bad_datastream_package_categories/data_stream/mylogs/agent/stream/stream.yml.hbs @@ -0,0 +1 @@ +{{fields "stream"}} diff --git a/test/packages/bad_datastream_package_categories/data_stream/mylogs/fields/fields.yml b/test/packages/bad_datastream_package_categories/data_stream/mylogs/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/bad_datastream_package_categories/data_stream/mylogs/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/bad_datastream_package_categories/data_stream/mylogs/manifest.yml b/test/packages/bad_datastream_package_categories/data_stream/mylogs/manifest.yml new file mode 100644 index 000000000..9b8b61a92 --- /dev/null +++ b/test/packages/bad_datastream_package_categories/data_stream/mylogs/manifest.yml @@ -0,0 +1,8 @@ +title: My logs +type: logs +categories: + - security +streams: + - input: logfile + title: My logs + description: Collect my logs diff --git a/test/packages/bad_datastream_package_categories/manifest.yml b/test/packages/bad_datastream_package_categories/manifest.yml new file mode 100644 index 000000000..87f63b1aa --- /dev/null +++ b/test/packages/bad_datastream_package_categories/manifest.yml @@ -0,0 +1,26 @@ +format_version: 3.6.0 +name: bad_datastream_package_categories +title: Bad package where package manifest is missing datastream parent categories +description: Package manifest categories do not include all parent-level categories present in the data stream manifests. +version: 0.0.1 +type: integration +categories: + - observability +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0' +policy_templates: + - name: mytemplate + title: My template + description: Collect my logs + data_streams: + - mylogs + inputs: + - type: logfile + title: Collect my logs + description: Collecting logs via log input +owner: + github: elastic/foobar + type: community diff --git a/test/packages/good_datastream_package_categories/changelog.yml b/test/packages/good_datastream_package_categories/changelog.yml new file mode 100644 index 000000000..27e62eecd --- /dev/null +++ b/test/packages/good_datastream_package_categories/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1 diff --git a/test/packages/good_datastream_package_categories/data_stream/mylogs/agent/stream/stream.yml.hbs b/test/packages/good_datastream_package_categories/data_stream/mylogs/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..ad7430eeb --- /dev/null +++ b/test/packages/good_datastream_package_categories/data_stream/mylogs/agent/stream/stream.yml.hbs @@ -0,0 +1 @@ +{{fields "stream"}} diff --git a/test/packages/good_datastream_package_categories/data_stream/mylogs/fields/fields.yml b/test/packages/good_datastream_package_categories/data_stream/mylogs/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_datastream_package_categories/data_stream/mylogs/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_datastream_package_categories/data_stream/mylogs/manifest.yml b/test/packages/good_datastream_package_categories/data_stream/mylogs/manifest.yml new file mode 100644 index 000000000..7c9fc4396 --- /dev/null +++ b/test/packages/good_datastream_package_categories/data_stream/mylogs/manifest.yml @@ -0,0 +1,8 @@ +title: My logs +type: logs +categories: + - observability +streams: + - input: logfile + title: My logs + description: Collect my logs diff --git a/test/packages/good_datastream_package_categories/manifest.yml b/test/packages/good_datastream_package_categories/manifest.yml new file mode 100644 index 000000000..c38391f07 --- /dev/null +++ b/test/packages/good_datastream_package_categories/manifest.yml @@ -0,0 +1,26 @@ +format_version: 3.6.0 +name: good_datastream_package_categories +title: Good package where package manifest includes all datastream parent categories +description: Package manifest categories include all parent-level categories present in the data stream manifests. +version: 0.0.1 +type: integration +categories: + - observability +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.0.0' +policy_templates: + - name: mytemplate + title: My template + description: Collect my logs + data_streams: + - mylogs + inputs: + - type: logfile + title: Collect my logs + description: Collecting logs via log input +owner: + github: elastic/foobar + type: community From e4109229904ebd35fb860308bacef24af9ac4a36 Mon Sep 17 00:00:00 2001 From: jdkurma Date: Wed, 4 Mar 2026 19:18:29 -0600 Subject: [PATCH 03/13] address feedback --- .../validate_datastream_package_categories.go | 45 ++++++++++++------- ...e_policy_template_datastream_categories.go | 40 ++++++++++++----- .../manifest.yml | 2 + .../manifest.yml | 2 + 4 files changed, 64 insertions(+), 25 deletions(-) diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories.go b/code/go/internal/validator/semantic/validate_datastream_package_categories.go index 0032ae4c2..689d2247a 100644 --- a/code/go/internal/validator/semantic/validate_datastream_package_categories.go +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories.go @@ -7,7 +7,6 @@ package semantic import ( "fmt" "io" - "io/fs" "net/http" "path" "slices" @@ -56,9 +55,31 @@ func fetchRegistryParentCategories() (map[string]struct{}, error) { return parentCategories, nil } -type packageManifestWithPackageCategories struct { - Type string `yaml:"type"` - Categories []string `yaml:"categories"` +func readPackageManifestTypeAndCategories(fsys fspath.FS) (string, []string, error) { + manifest, err := readManifest(fsys) + if err != nil { + return "", nil, err + } + + typeVal, err := manifest.Values("$.type") + if err != nil { + return "", nil, fmt.Errorf("can't read manifest type: %w", err) + } + pkgType, ok := typeVal.(string) + if !ok { + return "", nil, fmt.Errorf("manifest type is not a string") + } + + catsVal, err := manifest.Values("$.categories[*]") + if err != nil { + // categories field may be absent + return pkgType, nil, nil + } + cats, err := toStringSlice(catsVal) + if err != nil { + return "", nil, fmt.Errorf("can't read manifest categories: %w", err) + } + return pkgType, cats, nil } // ValidateDatastreamPackageCategories validates that the package manifest @@ -67,19 +88,13 @@ type packageManifestWithPackageCategories struct { // categories.yml. Data stream manifests without a categories field are skipped. func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationErrors { manifestPath := "manifest.yml" - data, err := fs.ReadFile(fsys, manifestPath) + pkgType, pkgCategories, err := readPackageManifestTypeAndCategories(fsys) if err != nil { return specerrors.ValidationErrors{ - specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToReadManifest)} - } - - var manifest packageManifestWithPackageCategories - if err := yaml.Unmarshal(data, &manifest); err != nil { - return specerrors.ValidationErrors{ - specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)} + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} } - if manifest.Type != packageTypeIntegration { + if pkgType != packageTypeIntegration { return nil } @@ -103,7 +118,7 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr for dsName, dsCats := range dsCategories { var missingCats []string for _, dsCat := range dsCats { - if _, isParent := parentCategories[dsCat]; isParent && !slices.Contains(manifest.Categories, dsCat) { + if _, isParent := parentCategories[dsCat]; isParent && !slices.Contains(pkgCategories, dsCat) { missingCats = append(missingCats, dsCat) } } @@ -114,7 +129,7 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr errs = append(errs, specerrors.NewStructuredErrorf( "file \"%s\" is invalid: package manifest categories %v are missing parent categories %v from data stream \"%s\" (defined in \"%s\")", fsys.Path(manifestPath), - manifest.Categories, + pkgCategories, missingCats, dsName, fsys.Path(dsManifestPath), diff --git a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go index b9a90d599..9c56ead72 100644 --- a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go +++ b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go @@ -30,6 +30,32 @@ type dataStreamManifestWithCategories struct { Categories []string `yaml:"categories"` } +func readPackageManifestPolicyTemplates(fsys fspath.FS) (string, []policyTemplateWithCategories, error) { + manifest, err := readManifest(fsys) + if err != nil { + return "", nil, err + } + + typeVal, err := manifest.Values("$.type") + if err != nil { + return "", nil, err + } + pkgType, ok := typeVal.(string) + if !ok { + return "", nil, nil + } + + data, err := manifest.ReadAll() + if err != nil { + return "", nil, err + } + var pkg packageManifestWithCategories + if err := yaml.Unmarshal(data, &pkg); err != nil { + return "", nil, err + } + return pkgType, pkg.PolicyTemplates, nil +} + // ValidatePolicyTemplateDatastreamCategories validates that when a policy template // entry in the package manifest.yml defines categories, those categories match the // categories defined in the manifest.yml of each referenced data stream. @@ -38,20 +64,14 @@ func ValidatePolicyTemplateDatastreamCategories(fsys fspath.FS) specerrors.Valid var errs specerrors.ValidationErrors manifestPath := "manifest.yml" - data, err := fs.ReadFile(fsys, manifestPath) + pkgType, policyTemplates, err := readPackageManifestPolicyTemplates(fsys) if err != nil { return specerrors.ValidationErrors{ - specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToReadManifest)} - } - - var manifest packageManifestWithCategories - if err := yaml.Unmarshal(data, &manifest); err != nil { - return specerrors.ValidationErrors{ - specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)} + specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} } // only validate integration type packages - if manifest.Type != packageTypeIntegration { + if pkgType != packageTypeIntegration { return nil } @@ -61,7 +81,7 @@ func ValidatePolicyTemplateDatastreamCategories(fsys fspath.FS) specerrors.Valid specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)} } - for _, pt := range manifest.PolicyTemplates { + for _, pt := range policyTemplates { // skip policy templates that don't define both categories and data_streams if len(pt.Categories) == 0 || len(pt.DataStreams) == 0 { continue diff --git a/test/packages/bad_datastream_categories_mismatch/manifest.yml b/test/packages/bad_datastream_categories_mismatch/manifest.yml index 74c120959..289861c1b 100644 --- a/test/packages/bad_datastream_categories_mismatch/manifest.yml +++ b/test/packages/bad_datastream_categories_mismatch/manifest.yml @@ -4,6 +4,8 @@ title: Bad package with mismatched datastream categories description: Policy template categories do not match the data stream manifest categories. version: 0.0.1 type: integration +categories: + - security source: license: "Apache-2.0" conditions: diff --git a/test/packages/good_datastream_categories_match/manifest.yml b/test/packages/good_datastream_categories_match/manifest.yml index 7284cfcd5..b2c6eaf1a 100644 --- a/test/packages/good_datastream_categories_match/manifest.yml +++ b/test/packages/good_datastream_categories_match/manifest.yml @@ -4,6 +4,8 @@ title: Good package with matching datastream categories description: Policy template categories match the data stream manifest categories. version: 0.0.1 type: integration +categories: + - observability source: license: "Apache-2.0" conditions: From 3d214a1d45267ff4c3115c956db08b8112e11a21 Mon Sep 17 00:00:00 2001 From: jdkurma Date: Wed, 4 Mar 2026 19:24:13 -0600 Subject: [PATCH 04/13] add docs --- test/packages/bad_datastream_package_categories/docs/README.md | 1 + test/packages/good_datastream_package_categories/docs/README.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 test/packages/bad_datastream_package_categories/docs/README.md create mode 100644 test/packages/good_datastream_package_categories/docs/README.md diff --git a/test/packages/bad_datastream_package_categories/docs/README.md b/test/packages/bad_datastream_package_categories/docs/README.md new file mode 100644 index 000000000..66173aec4 --- /dev/null +++ b/test/packages/bad_datastream_package_categories/docs/README.md @@ -0,0 +1 @@ +# Test package diff --git a/test/packages/good_datastream_package_categories/docs/README.md b/test/packages/good_datastream_package_categories/docs/README.md new file mode 100644 index 000000000..66173aec4 --- /dev/null +++ b/test/packages/good_datastream_package_categories/docs/README.md @@ -0,0 +1 @@ +# Test package From 5db2607316baddb35c7f3c80174a4cbbd3432288 Mon Sep 17 00:00:00 2001 From: jdkurma Date: Wed, 4 Mar 2026 20:25:17 -0600 Subject: [PATCH 05/13] address feedback --- ...e_policy_template_datastream_categories.go | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go index 9c56ead72..ebabb3057 100644 --- a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go +++ b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go @@ -5,13 +5,14 @@ package semantic import ( - "io/fs" + "fmt" "path" "sort" "gopkg.in/yaml.v3" "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/internal/pkgpath" "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) @@ -26,9 +27,6 @@ type packageManifestWithCategories struct { PolicyTemplates []policyTemplateWithCategories `yaml:"policy_templates"` } -type dataStreamManifestWithCategories struct { - Categories []string `yaml:"categories"` -} func readPackageManifestPolicyTemplates(fsys fspath.FS) (string, []policyTemplateWithCategories, error) { manifest, err := readManifest(fsys) @@ -42,7 +40,7 @@ func readPackageManifestPolicyTemplates(fsys fspath.FS) (string, []policyTemplat } pkgType, ok := typeVal.(string) if !ok { - return "", nil, nil + return "", nil, fmt.Errorf("manifest type is not a string: %v", typeVal) } data, err := manifest.ReadAll() @@ -116,31 +114,27 @@ func ValidatePolicyTemplateDatastreamCategories(fsys fspath.FS) specerrors.Valid // data_stream/*/manifest.yml and returns a map of data stream name to its categories. // Data streams without a categories field are omitted from the map. func readDataStreamManifestCategories(fsys fspath.FS) (map[string][]string, error) { - result := make(map[string][]string) - - manifests, err := fs.Glob(fsys, "data_stream/*/manifest.yml") + files, err := pkgpath.Files(fsys, "data_stream/*/manifest.yml") if err != nil { return nil, err } - for _, file := range manifests { - data, err := fs.ReadFile(fsys, file) + result := make(map[string][]string) + for _, f := range files { + catsVal, err := f.Values("$.categories[*]") if err != nil { - return nil, err + continue } - - var m dataStreamManifestWithCategories - if err := yaml.Unmarshal(data, &m); err != nil { - return nil, err + cats, err := toStringSlice(catsVal) + if err != nil { + return nil, fmt.Errorf("can't read categories from %s: %w", f.Path(), err) } - - if len(m.Categories) == 0 { + if len(cats) == 0 { continue } - // path is data_stream/{name}/manifest.yml — extract the name component - dsName := path.Base(path.Dir(file)) - result[dsName] = m.Categories + dsName := path.Base(path.Dir(f.Path())) + result[dsName] = cats } return result, nil From 84eb6316c2b901129a566b2cfaf8136cd11e81c7 Mon Sep 17 00:00:00 2001 From: jdkurma Date: Wed, 4 Mar 2026 20:42:47 -0600 Subject: [PATCH 06/13] lint --- .../semantic/validate_policy_template_datastream_categories.go | 1 - 1 file changed, 1 deletion(-) diff --git a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go index ebabb3057..de17da138 100644 --- a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go +++ b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go @@ -27,7 +27,6 @@ type packageManifestWithCategories struct { PolicyTemplates []policyTemplateWithCategories `yaml:"policy_templates"` } - func readPackageManifestPolicyTemplates(fsys fspath.FS) (string, []policyTemplateWithCategories, error) { manifest, err := readManifest(fsys) if err != nil { From 7f03b1fc62a4e8925a4401f07963680e013cc4f2 Mon Sep 17 00:00:00 2001 From: jdkurma Date: Wed, 4 Mar 2026 20:49:09 -0600 Subject: [PATCH 07/13] lint --- spec/changelog.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spec/changelog.yml b/spec/changelog.yml index cbe464d27..cf2f9e8f8 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -33,6 +33,9 @@ - description: Add support for dynamic_signal_types field in input package policy templates for OTel input type. type: enhancement link: https://github.com/elastic/package-spec/pull/1067 + - description: Allow optional _dev/scripts directory in integration, input, and content packages for development tooling files. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1090 # Pending on https://github.com/elastic/kibana/issues/252939 - description: Add support for multiple template paths in input packages, integration inputs, and data streams. type: enhancement @@ -41,6 +44,13 @@ - description: Add support for package dependencies in integration packages via requires field with input and content package types. type: enhancement link: https://github.com/elastic/package-spec/pull/1071 + # Pending on https://github.com/elastic/package-spec/issues/1085 + - description: Allow otelcol input type and dynamic_signal_types field in integration packages. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1091 + - description: Add deployer option to system benchmark configuration. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1099 - description: Add semantic validator to verify data stream manifest categories match policy template categories. type: enhancement link: https://github.com/elastic/package-spec/pull/1095 From f3d842b6a88b561c0eb4d2c9bddf6e4319edb1e9 Mon Sep 17 00:00:00 2001 From: jdkurma Date: Wed, 4 Mar 2026 22:20:44 -0600 Subject: [PATCH 08/13] check http status --- .../validate_datastream_package_categories.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories.go b/code/go/internal/validator/semantic/validate_datastream_package_categories.go index 689d2247a..ff6b21b56 100644 --- a/code/go/internal/validator/semantic/validate_datastream_package_categories.go +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories.go @@ -38,14 +38,22 @@ func fetchRegistryParentCategories() (map[string]struct{}, error) { } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected HTTP %d (%s) fetching %s", resp.StatusCode, resp.Status, packageRegistryCategoriesURL) + } + body, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read categories response: %w", err) + return nil, fmt.Errorf("failed to read categories response from %s: %w", packageRegistryCategoriesURL, err) } var rc registryCategories if err := yaml.Unmarshal(body, &rc); err != nil { - return nil, fmt.Errorf("failed to parse categories YAML: %w", err) + return nil, fmt.Errorf("failed to parse categories YAML from %s: %w", packageRegistryCategoriesURL, err) + } + + if len(rc.Categories) == 0 { + return nil, fmt.Errorf("no categories found in response from %s", packageRegistryCategoriesURL) } parentCategories := make(map[string]struct{}, len(rc.Categories)) From 96ed6b4edee5f8eb05628ffdbe2e0300171d5323 Mon Sep 17 00:00:00 2001 From: jdkurma Date: Mon, 13 Apr 2026 00:21:02 -0500 Subject: [PATCH 09/13] interpolate parent cat for subcat only ds --- .../validate_datastream_package_categories.go | 31 ++++++++---- ...date_datastream_package_categories_test.go | 21 +++++++- spec/changelog.yml | 48 ++++++++++++++----- 3 files changed, 79 insertions(+), 21 deletions(-) diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories.go b/code/go/internal/validator/semantic/validate_datastream_package_categories.go index ff6b21b56..88191bb4c 100644 --- a/code/go/internal/validator/semantic/validate_datastream_package_categories.go +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories.go @@ -30,7 +30,8 @@ type registryCategories struct { } `yaml:"categories"` } -func fetchRegistryParentCategories() (map[string]struct{}, error) { + +func fetchRegistryCategoryToParentMap() (map[string]string, error) { client := &http.Client{Timeout: 10 * time.Second} resp, err := client.Get(packageRegistryCategoriesURL) if err != nil { @@ -56,11 +57,14 @@ func fetchRegistryParentCategories() (map[string]struct{}, error) { return nil, fmt.Errorf("no categories found in response from %s", packageRegistryCategoriesURL) } - parentCategories := make(map[string]struct{}, len(rc.Categories)) - for id := range rc.Categories { - parentCategories[id] = struct{}{} + categoryToParent := make(map[string]string) + for parentID, cat := range rc.Categories { + categoryToParent[parentID] = parentID + for subID := range cat.Subcategories { + categoryToParent[subID] = parentID + } } - return parentCategories, nil + return categoryToParent, nil } func readPackageManifestTypeAndCategories(fsys fspath.FS) (string, []string, error) { @@ -91,7 +95,7 @@ func readPackageManifestTypeAndCategories(fsys fspath.FS) (string, []string, err } // ValidateDatastreamPackageCategories validates that the package manifest -// categories include all parent-level categories present in any data stream +// categories include all parent-level equivalent categories present in any data stream // manifest. Parent categories are determined by fetching the package registry // categories.yml. Data stream manifests without a categories field are skipped. func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationErrors { @@ -116,7 +120,7 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr return nil } - parentCategories, err := fetchRegistryParentCategories() + categoryToParent, err := fetchRegistryCategoryToParentMap() if err != nil { return specerrors.ValidationErrors{ specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to load registry categories: %w", fsys.Path(manifestPath), err)} @@ -124,10 +128,19 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr var errs specerrors.ValidationErrors for dsName, dsCats := range dsCategories { + seen := make(map[string]bool) var missingCats []string for _, dsCat := range dsCats { - if _, isParent := parentCategories[dsCat]; isParent && !slices.Contains(pkgCategories, dsCat) { - missingCats = append(missingCats, dsCat) + parent, known := categoryToParent[dsCat] + if !known { + continue + } + if seen[parent] { + continue + } + seen[parent] = true + if !slices.Contains(pkgCategories, parent) { + missingCats = append(missingCats, parent) } } diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories_test.go b/code/go/internal/validator/semantic/validate_datastream_package_categories_test.go index 0d3eb8b67..58a957005 100644 --- a/code/go/internal/validator/semantic/validate_datastream_package_categories_test.go +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories_test.go @@ -55,7 +55,7 @@ type: logs }, }, { - title: "datastream has subcategory only, no parent to enforce", + title: "datastream subcategory requires its parent in package", setup: func(t *testing.T, dir string) { writeManifest(t, dir, ` type: integration @@ -67,6 +67,25 @@ title: My Logs categories: - credential_management type: logs +`) + }, + expectedErrs: []string{ + `package manifest categories [observability] are missing parent categories [security] from data stream "mylogs"`, + }, + }, + { + title: "datastream subcategory parent already in package", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +categories: + - security +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +categories: + - credential_management +type: logs `) }, }, diff --git a/spec/changelog.yml b/spec/changelog.yml index cf2f9e8f8..a0bc9ed7f 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -2,12 +2,22 @@ ## This file documents changes in the package specification. It is NOT a package specification file. ## Newer entries go at the bottom of each in-development version. ## -- version: 3.6.0-next +- version: 3.7.0-next changes: # Pending on https://github.com/elastic/kibana/issues/220294 - description: Add support for semantic_text field definition. type: enhancement link: https://github.com/elastic/package-spec/pull/807 +- version: 3.6.1-next + changes: + - description: Add var_groups support to policy template and input levels in integration packages, and to policy template and package root levels in input packages. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1120 + - description: Add semantic validator to verify data stream manifest categories match policy template categories. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1095 +- version: 3.6.0 + changes: - description: Add pipeline tag validations. type: breaking-change link: https://github.com/elastic/package-spec/pull/1010 @@ -21,22 +31,15 @@ - description: Add support for deprecating packages or individual features (policy_templates, inputs, data_streams or variables). type: enhancement link: https://github.com/elastic/package-spec/pull/1053 - # Pending on https://github.com/elastic/kibana/pull/249449 - # Pending on https://github.com/elastic/integrations/pull/16985 - description: Add var_groups schema to support conditional variable groups for Cloud Connector integration. type: enhancement link: https://github.com/elastic/package-spec/issues/1054 - # Pending on https://github.com/elastic/kibana/pull/251205 - description: Allow to set time series index mode in input packages. type: enhancement link: https://github.com/elastic/package-spec/pull/1066 - description: Add support for dynamic_signal_types field in input package policy templates for OTel input type. type: enhancement link: https://github.com/elastic/package-spec/pull/1067 - - description: Allow optional _dev/scripts directory in integration, input, and content packages for development tooling files. - type: enhancement - link: https://github.com/elastic/package-spec/pull/1090 - # Pending on https://github.com/elastic/kibana/issues/252939 - description: Add support for multiple template paths in input packages, integration inputs, and data streams. type: enhancement link: https://github.com/elastic/package-spec/pull/1089 @@ -44,16 +47,39 @@ - description: Add support for package dependencies in integration packages via requires field with input and content package types. type: enhancement link: https://github.com/elastic/package-spec/pull/1071 - # Pending on https://github.com/elastic/package-spec/issues/1085 + # Pending on https://github.com/elastic/kibana/issues/252949 - description: Allow otelcol input type and dynamic_signal_types field in integration packages. type: enhancement link: https://github.com/elastic/package-spec/pull/1091 + - description: Add support for migrating input types at integrations policy templates and data streams. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1021 + - description: Add support for `profiles` policy template type in input packages + type: enhancement + link: https://github.com/elastic/package-spec/pull/1092 + - description: Support including multiple sample events + type: enhancement + link: https://github.com/elastic/package-spec/pull/582 + - description: Align integration template path semantic validation with Fleet + type: bugfix + link: https://github.com/elastic/package-spec/pull/1122 +- version: 3.5.8 + changes: - description: Add deployer option to system benchmark configuration. type: enhancement link: https://github.com/elastic/package-spec/pull/1099 - - description: Add semantic validator to verify data stream manifest categories match policy template categories. + - description: Allow to force the format used in the policy API in system and policy tests. type: enhancement - link: https://github.com/elastic/package-spec/pull/1095 + link: https://github.com/elastic/package-spec/pull/1103 + - description: Add support for auto_expand_replicas, max_result_window, and refresh_interval index settings in data stream templates. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1092 + - description: Add deployer option to system test configuration. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1104 + - description: Allow optional _dev/scripts directory in integration, input, and content packages for development tooling files. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1090 - version: 3.5.7 changes: - description: Allow _dev directory for content-only packages; use _dev/shared for development files (e.g. dashboard YML sources). From d694064eaa1f3eb263b50fbeca93fc92dd1e459b Mon Sep 17 00:00:00 2001 From: JD Kurma Date: Mon, 13 Apr 2026 00:28:32 -0500 Subject: [PATCH 10/13] format --- .../validator/semantic/validate_datastream_package_categories.go | 1 - 1 file changed, 1 deletion(-) diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories.go b/code/go/internal/validator/semantic/validate_datastream_package_categories.go index 88191bb4c..5d36cab82 100644 --- a/code/go/internal/validator/semantic/validate_datastream_package_categories.go +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories.go @@ -30,7 +30,6 @@ type registryCategories struct { } `yaml:"categories"` } - func fetchRegistryCategoryToParentMap() (map[string]string, error) { client := &http.Client{Timeout: 10 * time.Second} resp, err := client.Get(packageRegistryCategoriesURL) From f0c1574a2460473dfa7c5c360f58940b57820440 Mon Sep 17 00:00:00 2001 From: JD Kurma Date: Tue, 21 Apr 2026 21:04:44 -0500 Subject: [PATCH 11/13] raise err on invalid categories --- .../semantic/validate_datastream_package_categories.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories.go b/code/go/internal/validator/semantic/validate_datastream_package_categories.go index 5d36cab82..d75a5f56c 100644 --- a/code/go/internal/validator/semantic/validate_datastream_package_categories.go +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories.go @@ -132,7 +132,14 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr for _, dsCat := range dsCats { parent, known := categoryToParent[dsCat] if !known { - continue + dsManifestPath := path.Join("data_stream", dsName, "manifest.yml") + errs = append(errs, specerrors.NewStructuredErrorf( + "file \"%s\" is invalid: data stream \"%s\" has unrecognized category \"%s\" (defined in \"%s\")", + fsys.Path(manifestPath), + dsName, + dsCat, + fsys.Path(dsManifestPath), + )) } if seen[parent] { continue From a4cb6103458bc40f553d4be921b4bc7bf49caa9e Mon Sep 17 00:00:00 2001 From: jdkurma Date: Tue, 12 May 2026 12:26:25 -0500 Subject: [PATCH 12/13] switch to contains rather than strict equality between policy and ds mani cats --- .../validate_datastream_package_categories.go | 6 +-- ...e_policy_template_datastream_categories.go | 40 +++++++++---------- ...icy_template_datastream_categories_test.go | 24 ++++++++++- code/go/pkg/validator/validator_test.go | 2 +- 4 files changed, 42 insertions(+), 30 deletions(-) diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories.go b/code/go/internal/validator/semantic/validate_datastream_package_categories.go index d75a5f56c..3807f9cf2 100644 --- a/code/go/internal/validator/semantic/validate_datastream_package_categories.go +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories.go @@ -127,7 +127,6 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr var errs specerrors.ValidationErrors for dsName, dsCats := range dsCategories { - seen := make(map[string]bool) var missingCats []string for _, dsCat := range dsCats { parent, known := categoryToParent[dsCat] @@ -140,12 +139,9 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr dsCat, fsys.Path(dsManifestPath), )) - } - if seen[parent] { continue } - seen[parent] = true - if !slices.Contains(pkgCategories, parent) { + if !slices.Contains(pkgCategories, parent) && !slices.Contains(missingCats, parent) { missingCats = append(missingCats, parent) } } diff --git a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go index de17da138..dec6f3643 100644 --- a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go +++ b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories.go @@ -7,6 +7,7 @@ package semantic import ( "fmt" "path" + "slices" "sort" "gopkg.in/yaml.v3" @@ -54,8 +55,9 @@ func readPackageManifestPolicyTemplates(fsys fspath.FS) (string, []policyTemplat } // ValidatePolicyTemplateDatastreamCategories validates that when a policy template -// entry in the package manifest.yml defines categories, those categories match the -// categories defined in the manifest.yml of each referenced data stream. +// entry in the package manifest.yml defines categories, each referenced data stream's +// manifest.yml categories include all of the policy template's categories. Data streams +// may declare additional categories beyond what the policy template specifies. // Data stream manifests without a categories field are skipped. func ValidatePolicyTemplateDatastreamCategories(fsys fspath.FS) specerrors.ValidationErrors { var errs specerrors.ValidationErrors @@ -91,15 +93,16 @@ func ValidatePolicyTemplateDatastreamCategories(fsys fspath.FS) specerrors.Valid continue } - if !categoriesEqual(pt.Categories, dsCats) { + missing := missingCategories(pt.Categories, dsCats) + if len(missing) > 0 { dsManifestPath := path.Join("data_stream", dsName, "manifest.yml") errs = append(errs, specerrors.NewStructuredErrorf( - "file \"%s\" is invalid: policy template \"%s\" categories %v do not match data stream \"%s\" manifest categories %v (defined in \"%s\")", + "file \"%s\" is invalid: data stream \"%s\" manifest categories %v are missing policy template \"%s\" categories %v (defined in \"%s\")", fsys.Path(manifestPath), - pt.Name, - pt.Categories, dsName, dsCats, + pt.Name, + missing, fsys.Path(dsManifestPath), )) } @@ -139,22 +142,15 @@ func readDataStreamManifestCategories(fsys fspath.FS) (map[string][]string, erro return result, nil } -// categoriesEqual returns true if both slices contain exactly the same set of -// categories, regardless of order. -func categoriesEqual(a, b []string) bool { - if len(a) != len(b) { - return false - } - aCopy := make([]string, len(a)) - bCopy := make([]string, len(b)) - copy(aCopy, a) - copy(bCopy, b) - sort.Strings(aCopy) - sort.Strings(bCopy) - for i := range aCopy { - if aCopy[i] != bCopy[i] { - return false +// missingCategories returns the categories present in want but absent from have. +// The result is sorted for deterministic error output. +func missingCategories(want, have []string) []string { + var missing []string + for _, c := range want { + if !slices.Contains(have, c) { + missing = append(missing, c) } } - return true + sort.Strings(missing) + return missing } diff --git a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go index b5d069972..d60c5ebd0 100644 --- a/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go +++ b/code/go/internal/validator/semantic/validate_policy_template_datastream_categories_test.go @@ -58,7 +58,28 @@ categories: }, }, { - title: "categories mismatch", + title: "data stream missing policy template category", + setup: func(t *testing.T, dir string) { + writeManifest(t, dir, ` +type: integration +policy_templates: + - name: mytemplate + data_streams: + - mylogs + categories: + - observability + - network +`) + writeDataStreamManifest(t, dir, "mylogs", ` +title: My Logs +categories: + - observability +`) + }, + expectedErrs: []string{`policy template "mytemplate"`, `"mylogs"`, `[network]`}, + }, + { + title: "data stream has extra categories beyond policy template", setup: func(t *testing.T, dir string) { writeManifest(t, dir, ` type: integration @@ -76,7 +97,6 @@ categories: - security `) }, - expectedErrs: []string{`policy template "mytemplate"`, `"mylogs"`}, }, { title: "data stream manifest has no categories field", diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index 92d62c5f6..a0cbd873a 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -152,7 +152,7 @@ func TestValidateFile(t *testing.T) { "bad_datastream_categories_mismatch": { "manifest.yml", []string{ - fmt.Sprintf(`policy template "mytemplate" categories [observability] do not match data stream "mylogs" manifest categories [security] (defined in "%sbad_datastream_categories_mismatch/data_stream/mylogs/manifest.yml")`, osTestBasePath), + fmt.Sprintf(`data stream "mylogs" manifest categories [security] are missing policy template "mytemplate" categories [observability] (defined in "%sbad_datastream_categories_mismatch/data_stream/mylogs/manifest.yml")`, osTestBasePath), }, }, "bad_datastream_package_categories": { From 5a71d6cb6ed18c07fb1b7e9f9d31ef8ced724fe8 Mon Sep 17 00:00:00 2001 From: JD Kurma Date: Tue, 12 May 2026 12:27:18 -0500 Subject: [PATCH 13/13] Update code/go/internal/validator/semantic/validate_datastream_package_categories.go Co-authored-by: Jaime Soriano Pastor --- .../semantic/validate_datastream_package_categories.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/internal/validator/semantic/validate_datastream_package_categories.go b/code/go/internal/validator/semantic/validate_datastream_package_categories.go index 3807f9cf2..0a4de7638 100644 --- a/code/go/internal/validator/semantic/validate_datastream_package_categories.go +++ b/code/go/internal/validator/semantic/validate_datastream_package_categories.go @@ -19,7 +19,7 @@ import ( "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" ) -const packageRegistryCategoriesURL = "https://raw.githubusercontent.com/elastic/package-registry/main/categories/categories.yml" +const packageRegistryCategoriesURL = "https://raw.githubusercontent.com/elastic/package-registry/v1.38.0/categories/categories.yml" type registryCategories struct { Categories map[string]struct {