Validation rules: build/source mode enforcement

code/go/internal/validator/semantic/types.go

@@ -352,9 +352,11 @@ func listDataStreams(fsys fspath.FS) ([]string, error) {
 		return nil, fmt.Errorf("can't list data streams directory: %w", err)
 	}
 
-	list := make([]string, len(dataStreams))
-	for i, dataStream := range dataStreams {
-		list[i] = dataStream.Name()
+	var list []string
+	for _, dataStream := range dataStreams {
+		if dataStream.IsDir() {
+			list = append(list, dataStream.Name())
+		}
 	}
 	return list, nil
 }

code/go/internal/validator/semantic/validate_datastream_package_categories.go

@@ -105,7 +105,7 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr
 			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)}
 	}
 
-	if pkgType != packageTypeIntegration {
+	if pkgType != integrationPackageType {
 		return nil
 	}
 

code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go

@@ -47,7 +47,7 @@ func ValidateIntegrationInputsDeprecation(fsys fspath.FS) specerrors.ValidationE
 			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)}
 	}
 	// skip if not an integration package
-	if m.Type != packageTypeIntegration {
+	if m.Type != integrationPackageType {
 		return nil
 	}
 

code/go/internal/validator/semantic/validate_integration_policy_template_path.go

@@ -20,11 +20,11 @@ import (
 
 const (
 	defaultStreamTemplatePath = "stream.yml.hbs"
-	packageTypeIntegration    = "integration"
 )
 
 type policyTemplateInput struct {
 	Type          string   `yaml:"type"`
+	Package       string   `yaml:"package"`
 	TemplatePath  string   `yaml:"template_path"`
 	TemplatePaths []string `yaml:"template_paths"`
 }
@@ -41,6 +41,7 @@ type integrationPackageManifest struct { // package manifest
 
 type stream struct {
 	Input         string   `yaml:"input"`
+	Package       string   `yaml:"package"`
 	TemplatePath  string   `yaml:"template_path"`
 	TemplatePaths []string `yaml:"template_paths"`
 }
@@ -78,7 +79,7 @@ func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErr
 			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)}
 	}
 
-	if manifest.Type != packageTypeIntegration {
+	if manifest.Type != integrationPackageType {
 		return nil
 	}
 
@@ -110,6 +111,14 @@ func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErr
 // under agent/input when template_paths or template_path is set (Fleet: template_paths first).
 func validateIntegrationPolicyTemplateInputs(fsys fspath.FS, policyTemplate integrationPolicyTemplate) error {
 	for _, input := range policyTemplate.Inputs {
+		// Composable inputs reference an input package via 'package:'. When no
+		// explicit template_path/template_paths is set, all templates come from
+		// the dependency and are only present after build. Skip those.
+		// If the composable input defines its own template_path or template_paths
+		// (overlay templates that live in the source package), those are validated.
+		if input.Package != "" && input.TemplatePath == "" && len(input.TemplatePaths) == 0 {
+			continue
+		}
 		if len(input.TemplatePaths) > 0 {
 			for _, tp := range input.TemplatePaths {
 				if err := validateAgentInputTemplatePath(fsys, tp); err != nil {
@@ -141,6 +150,17 @@ func validateAllDataStreamStreamTemplates(fsys fspath.FS, dsMap map[string]dataS
 		dsManifestPath := path.Join(dsDir, "manifest.yml")
 		manifest := dsMap[dsDir]
 		for _, s := range manifest.Streams {
+			// Composable streams reference an input package via 'package:'. When
+			// no explicit template_path/template_paths is set on the stream, all
+			// templates come from the dependency and are only present after build.
+			// Skip those — ValidateStreamInputMaterialized enforces that 'package:'
+			// is replaced by 'input:' in build mode.
+			// However, if the composable stream defines its own template_path or
+			// template_paths, those files must exist in the source package and are
+			// validated here.
+			if s.Package != "" && s.TemplatePath == "" && len(s.TemplatePaths) == 0 {
+				continue
+			}
 			if err := validateSingleDataStreamStreamTemplates(fsys, dsDir, s); err != nil {
 				errs = append(errs, specerrors.NewStructuredErrorf(
 					"file \"%s\" is invalid: data stream \"%s\" stream input %q: %w",

code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go

@@ -254,6 +254,65 @@ streams:
 		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
 		require.Empty(t, errs)
 	})
+
+	t.Run("composable stream with no explicit templates skips validation", func(t *testing.T) {
+		d := t.TempDir()
+		writeMinimalIntegrationManifest(t, d)
+		// No agent/stream directory — templates come entirely from the input package.
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "logs"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "manifest.yml"), []byte(`
+streams:
+  - package: some_input_package
+    title: Composable
+    description: d
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Empty(t, errs)
+	})
+
+	t.Run("composable stream with explicit template_paths validates those files", func(t *testing.T) {
+		d := t.TempDir()
+		writeMinimalIntegrationManifest(t, d)
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "logs", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", "overlay.yml.hbs"), []byte(`x`), 0o644)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "manifest.yml"), []byte(`
+streams:
+  - package: some_input_package
+    title: Composable with overlay
+    description: d
+    template_paths:
+      - overlay.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Empty(t, errs)
+	})
+
+	t.Run("composable stream with explicit template_paths fails when file missing", func(t *testing.T) {
+		d := t.TempDir()
+		writeMinimalIntegrationManifest(t, d)
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "logs", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "manifest.yml"), []byte(`
+streams:
+  - package: some_input_package
+    title: Composable with overlay
+    description: d
+    template_paths:
+      - missing.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Len(t, errs, 1)
+		require.Contains(t, errs[0].Error(), "template file not found")
+	})
 }
 func TestValidateIntegrationPolicyTemplates_NonIntegrationType(t *testing.T) {
 	d := t.TempDir()
@@ -323,6 +382,70 @@ streams:
 	errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
 	require.Empty(t, errs)
 }
+func TestValidateIntegrationPolicyTemplates_ComposableInputs(t *testing.T) {
+	t.Run("composable input with no templates skips validation", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - name: pt
+    inputs:
+      - package: some_input_package
+        title: Composable
+        description: d
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Empty(t, errs)
+	})
+
+	t.Run("composable input with explicit template_paths validates those files", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "agent", "input", "overlay.yml.hbs"), []byte(`x`), 0o644)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - name: pt
+    inputs:
+      - package: some_input_package
+        title: Composable with overlay
+        description: d
+        template_paths:
+          - overlay.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Empty(t, errs)
+	})
+
+	t.Run("composable input with explicit template_paths fails when file missing", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - name: pt
+    inputs:
+      - package: some_input_package
+        title: Composable with overlay
+        description: d
+        template_paths:
+          - missing.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Len(t, errs, 1)
+		require.Contains(t, errs[0].Error(), "template file not found")
+	})
+}
+
 func TestFindPathAtDirectory(t *testing.T) {
 	d := t.TempDir()
 

code/go/internal/validator/semantic/validate_no_dev_folder.go

@@ -0,0 +1,39 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"io/fs"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+// ValidateNoDevFolder errors for any _dev/ directory found in the package.
+// _dev/ is a source-only artifact used during development (tests, deploy
+// configs, build manifests). It must not appear in a built package that is
+// validated with ModeBuild.
+func ValidateNoDevFolder(fsys fspath.FS) specerrors.ValidationErrors {
+	var errs specerrors.ValidationErrors
+	walkErr := fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() && d.Name() == "_dev" {
+			errs = append(errs, specerrors.NewStructuredErrorf(
+				"file %q: _dev directory is not allowed in built packages",
+				fsys.Path(p),
+			))
+			// Skip the subtree to avoid generating child errors for each
+			// file inside the _dev directory.
+			return fs.SkipDir
+		}
+		return nil
+	})
+	if walkErr != nil {
+		errs = append(errs, specerrors.NewStructuredError(walkErr, specerrors.UnassignedCode))
+	}
+	return errs
+}

code/go/internal/validator/semantic/validate_no_dev_folder_test.go

@@ -0,0 +1,87 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+)
+
+func TestValidateNoDevFolder(t *testing.T) {
+	tests := []struct {
+		name          string
+		dirs          []string // directories to create relative to the temp root
+		expectErrors  bool
+		errorContains []string
+	}{
+		{
+			name:         "no _dev directory",
+			dirs:         []string{"data_stream/foo/fields"},
+			expectErrors: false,
+		},
+		{
+			name:          "_dev at package root",
+			dirs:          []string{"_dev/build"},
+			expectErrors:  true,
+			errorContains: []string{"_dev", "_dev directory is not allowed in built packages"},
+		},
+		{
+			name:          "_dev inside data_stream",
+			dirs:          []string{"data_stream/foo/_dev/test"},
+			expectErrors:  true,
+			errorContains: []string{"_dev", "_dev directory is not allowed in built packages"},
+		},
+		{
+			name: "multiple _dev directories",
+			dirs: []string{
+				"_dev/build",
+				"data_stream/foo/_dev/test",
+			},
+			expectErrors:  true,
+			errorContains: []string{"_dev directory is not allowed in built packages"},
+		},
+		{
+			name:         "directory named _devtools is not rejected",
+			dirs:         []string{"_devtools"},
+			expectErrors: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			tempDir := t.TempDir()
+
+			for _, d := range tc.dirs {
+				require.NoError(t, os.MkdirAll(filepath.Join(tempDir, d), 0o755))
+			}
+
+			fsys := fspath.DirFS(tempDir)
+			errs := ValidateNoDevFolder(fsys)
+
+			if !tc.expectErrors {
+				assert.Nil(t, errs, "expected no errors but got: %v", errs)
+				return
+			}
+
+			require.NotNil(t, errs, "expected validation errors but got none")
+			var sb strings.Builder
+			for _, e := range errs {
+				sb.WriteString(e.Error())
+				sb.WriteString("\n")
+			}
+			combined := sb.String()
+			for _, substr := range tc.errorContains {
+				assert.Contains(t, combined, substr)
+			}
+		})
+	}
+}