diff --git a/docs/modules/ROOT/pages/running/running-cli.adoc b/docs/modules/ROOT/pages/running/running-cli.adoc index 67c8d14d9c..4d5fb9293b 100644 --- a/docs/modules/ROOT/pages/running/running-cli.adoc +++ b/docs/modules/ROOT/pages/running/running-cli.adoc @@ -87,6 +87,38 @@ status: {} ``` This can be saved for future processing (ie, stored to a GIT repository and later deployed to a cluster via some GitOps deployment strategy). Consider that any **modeline** option will be translated accordingly. +Dry run is also the easiest way to customize per-source native build behavior. For example, if you are building a native executable and one of the source files should be embedded as a classpath resource instead of being interpreted as a Camel route, you can: + +1. Generate the Integration manifest with `kamel run ... -o yaml` +2. Edit the relevant `spec.sources[]` entry +3. Set `nativeImage: resource` +4. Apply the resulting manifest with `kubectl apply -f` + +For example: + +[source,console] +---- +kamel run Hello.java app-config.yaml -t quarkus.build-mode=native -o yaml +---- + +Then edit the generated Integration so the resource entry looks like: + +[source,yaml] +---- +spec: + sources: + - name: Hello.java + content: | + ... + - name: app-config.yaml + nativeImage: resource + content: | + greeting: hello + audience: native +---- + +This is particularly useful for files such as `.yaml` that may otherwise be inferred as Camel DSL routes. + [[modeline]] == Camel K Modeline @@ -226,4 +258,4 @@ kamel run \ https://gist.githubusercontent.com/${user-id}/${gist-id}/raw/${...}/routes.yaml ---- -NOTE: GitHub applies rate limiting to its APIs and as Authenticated requests get a higher rate limit, the `kamel` honour the env var GITHUB_TOKEN and if it is found, then it is used for GitHub authentication. \ No newline at end of file +NOTE: GitHub applies rate limiting to its APIs and as Authenticated requests get a higher rate limit, the `kamel` honour the env var GITHUB_TOKEN and if it is found, then it is used for GitHub authentication. diff --git a/docs/modules/ROOT/pages/running/running.adoc b/docs/modules/ROOT/pages/running/running.adoc index bf64628568..0da4ffaf46 100644 --- a/docs/modules/ROOT/pages/running/running.adoc +++ b/docs/modules/ROOT/pages/running/running.adoc @@ -92,6 +92,58 @@ spec: You can see the specification is a lot neater, so, try choosing Yaml DSL whenever it's possible. +== Native builds and classpath resources + +When you build a native executable, Camel K has to decide whether each entry in `spec.sources` is: + +- a Camel route that must be loaded by Camel +- a classpath resource that must be embedded into the native executable + +By default Camel K infers this from the source language or file extension. You can override that behavior explicitly with `spec.sources[].nativeImage`. + +This is useful when a file has a known Camel DSL extension, such as `.yaml`, but it is actually application data and not a route. + +[source,yaml] +---- +apiVersion: camel.apache.org/v1 +kind: Integration +metadata: + name: native-resource-example +spec: + sources: + - name: Hello.java + content: | + import org.apache.camel.builder.RouteBuilder; + + public class Hello extends RouteBuilder { + @Override + public void configure() throws Exception { + from("timer:tick?period=3000") + .setBody().constant("Hello from Camel K") + .log("${body}"); + } + } + - name: files/app-config.yaml + nativeImage: resource + content: | + greeting: hello + audience: native + traits: + quarkus: + buildMode: + - native +---- + +The `nativeImage` field accepts: + +- `auto`: Camel K infers the behavior. This is the default. +- `route`: force the source to be treated as a Camel route. +- `resource`: force the source to be packaged as a classpath resource. + +For example, if you really want a YAML file to stay a route in a native build, omit the field or set `nativeImage: route`. If you want the same kind of file to be embedded as application data, set `nativeImage: resource`. + +NOTE: `spec.sources[].nativeImage` is about files stored in the Integration itself. It does not replace runtime-mounted ConfigMaps or Secrets configured with xref:traits:mount.adoc[mount trait]. + == Runtime provider Camel K was originally equipped with a dedicated runtime known as Camel K Runtime. This is a lightweight layer on top of Camel Quarkus. However, you can directly run plain regular Camel Quarkus runtime applications as well. You will learn the concept of traits later on. For now, just be aware that you can run any Integration setting the plain Quarkus runtime using `camel` trait configuration. Here an example of how that would be: diff --git a/docs/modules/traits/pages/quarkus.adoc b/docs/modules/traits/pages/quarkus.adoc index 09a13a6532..210b22435e 100755 --- a/docs/modules/traits/pages/quarkus.adoc +++ b/docs/modules/traits/pages/quarkus.adoc @@ -66,3 +66,62 @@ NOTE: the variable names are "snake case" if you're using in `kamel` CLI, for ex // End of autogenerated code - DO NOT EDIT! (configuration) + +== Native source inclusion + +When Camel K builds a native executable, it needs to decide how each entry in `Integration.spec.sources` should be materialized. + +By default Camel K uses the source file name and language inference: + +- Camel DSL sources are treated as routes +- non-DSL files are treated as classpath resources + +You can now override this behavior explicitly on each `spec.sources[]` entry with the `nativeImage` field: + +[cols="1m,4a"] +|=== +|Value | Description + +|auto +|Use Camel K default inference. This is the default when the field is omitted. + +|route +|Force the source to be treated as a Camel route during native build. + +|resource +|Force the source to be packaged as a classpath resource during native build. +|=== + +This is especially useful when a file uses a known Camel DSL extension such as `.yaml`, but the file is application data and must be available on the classpath of the native executable. + +[source,yaml] +---- +apiVersion: camel.apache.org/v1 +kind: Integration +metadata: + name: native-resource-example +spec: + sources: + - name: routes/route.yaml + content: | + - from: + uri: "timer:tick" + steps: + - log: "Hello from Camel K" + - name: files/app-config.yaml + nativeImage: resource + content: | + greeting: hello + audience: native + traits: + quarkus: + buildMode: + - native +---- + +In the example above: + +- `routes/route.yaml` is still treated as a Camel route +- `files/app-config.yaml` is packaged into the native image as a classpath resource + +NOTE: `spec.sources[].nativeImage` only affects how Camel K packages entries from `spec.sources` during a native build. It is different from `mount.resources`, which mounts ConfigMaps or Secrets into the running Pod at runtime. diff --git a/pkg/apis/camel/v1/common_types.go b/pkg/apis/camel/v1/common_types.go index 1a7cf947a5..072a42c6b4 100644 --- a/pkg/apis/camel/v1/common_types.go +++ b/pkg/apis/camel/v1/common_types.go @@ -483,6 +483,9 @@ type SourceSpec struct { // specify which is the language (Camel DSL) used to interpret this source code Language Language `json:"language,omitempty"` + // Controls how the source is included when building a native executable. + // `auto` uses Camel K inference, `route` forces route inclusion, `resource` forces classpath resource inclusion. + NativeImage NativeImageSourceType `json:"nativeImage,omitempty"` // Loader is an optional id of the org.apache.camel.k.RoutesLoader that will // interpret this source at runtime Loader string `json:"loader,omitempty"` @@ -511,6 +514,19 @@ const ( SourceTypeErrorHandler SourceType = "errorHandler" ) +// NativeImageSourceType determines how a source should be materialized in a native build. +// +kubebuilder:validation:Enum=auto;route;resource +type NativeImageSourceType string + +const ( + // NativeImageSourceTypeAuto lets Camel K infer whether the source is a route or a resource. + NativeImageSourceTypeAuto NativeImageSourceType = "auto" + // NativeImageSourceTypeRoute forces the source to be treated as a Camel route during native build. + NativeImageSourceTypeRoute NativeImageSourceType = "route" + // NativeImageSourceTypeResource forces the source to be treated as a classpath resource during native build. + NativeImageSourceTypeResource NativeImageSourceType = "resource" +) + // DataSpec represents the way the source is materialized in the running `Pod`. type DataSpec struct { // the name of the specification diff --git a/pkg/apis/camel/v1/common_types_support.go b/pkg/apis/camel/v1/common_types_support.go index 2e590d0448..ad002ce055 100644 --- a/pkg/apis/camel/v1/common_types_support.go +++ b/pkg/apis/camel/v1/common_types_support.go @@ -278,6 +278,35 @@ func (s *SourceSpec) InferLanguage() Language { return "" } +func (s *SourceSpec) NativeImageSourceType() NativeImageSourceType { + if s.NativeImage == "" { + return NativeImageSourceTypeAuto + } + + return s.NativeImage +} + +func (s *SourceSpec) NativeImageAsResource() bool { + switch s.NativeImageSourceType() { + case NativeImageSourceTypeResource: + return true + case NativeImageSourceTypeRoute: + return false + default: + return s.InferLanguage() == "" + } +} + +func (s *SourceSpec) NativeImageAsRoute() bool { + switch s.NativeImageSourceType() { + case NativeImageSourceTypeRoute: + return true + case NativeImageSourceTypeResource: + return false + default: + return s.InferLanguage() != "" + } +} // Validate checks if the strategy is supported. func (b BuildStrategy) Validate() error { diff --git a/pkg/apis/camel/v1/integration_types_support_test.go b/pkg/apis/camel/v1/integration_types_support_test.go index 157b2b635e..f77c3876c7 100644 --- a/pkg/apis/camel/v1/integration_types_support_test.go +++ b/pkg/apis/camel/v1/integration_types_support_test.go @@ -61,6 +61,62 @@ func TestLanguageAlreadySet(t *testing.T) { assert.Equal(t, LanguageJavaSource, code.InferLanguage()) } +func TestNativeImageSourceTypeDefaultsToAuto(t *testing.T) { + code := SourceSpec{ + DataSpec: DataSpec{ + Name: "request.yaml", + }, + } + + assert.Equal(t, NativeImageSourceTypeAuto, code.NativeImageSourceType()) + assert.True(t, code.NativeImageAsRoute()) + assert.False(t, code.NativeImageAsResource()) +} + +func TestNativeImageSourceTypeCanForceResource(t *testing.T) { + code := SourceSpec{ + DataSpec: DataSpec{ + Name: "request.yaml", + }, + NativeImage: NativeImageSourceTypeResource, + } + + assert.Equal(t, NativeImageSourceTypeResource, code.NativeImageSourceType()) + assert.False(t, code.NativeImageAsRoute()) + assert.True(t, code.NativeImageAsResource()) +} + +func TestNativeImageSourceTypeCanForceRoute(t *testing.T) { + code := SourceSpec{ + DataSpec: DataSpec{ + Name: "notes.txt", + }, + NativeImage: NativeImageSourceTypeRoute, + } + + assert.Equal(t, NativeImageSourceTypeRoute, code.NativeImageSourceType()) + assert.True(t, code.NativeImageAsRoute()) + assert.False(t, code.NativeImageAsResource()) +} + +func TestNativeImageSourceTypeAutoAndEmptyAreEquivalent(t *testing.T) { + implicit := SourceSpec{ + DataSpec: DataSpec{ + Name: "request.yaml", + }, + } + explicit := SourceSpec{ + DataSpec: DataSpec{ + Name: "request.yaml", + }, + NativeImage: NativeImageSourceTypeAuto, + } + + assert.Equal(t, implicit.NativeImageSourceType(), explicit.NativeImageSourceType()) + assert.Equal(t, implicit.NativeImageAsRoute(), explicit.NativeImageAsRoute()) + assert.Equal(t, implicit.NativeImageAsResource(), explicit.NativeImageAsResource()) +} + func TestAddDependency(t *testing.T) { integration := IntegrationSpec{} integration.AddDependency("camel:file") diff --git a/pkg/builder/quarkus.go b/pkg/builder/quarkus.go index 9aafa5e63a..edcf5a1693 100644 --- a/pkg/builder/quarkus.go +++ b/pkg/builder/quarkus.go @@ -78,7 +78,24 @@ func prepareProjectWithSources(ctx *builderContext) error { } sourceList := "" + resourceList := make([]string, 0) for _, source := range ctx.Build.Sources { + if source.NativeImageAsResource() { + resourcePath := filepath.Join(ctx.Path, "maven", "src", "main", "resources", source.Name) + if err := os.MkdirAll(filepath.Dir(resourcePath), os.ModePerm); err != nil { + return fmt.Errorf("failure while creating resource folder: %w", err) + } + if err := os.WriteFile( + resourcePath, + []byte(source.Content), + projectModePerm, + ); err != nil { + return fmt.Errorf("failure while writing %s: %w", source.Name, err) + } + resourceList = append(resourceList, filepath.ToSlash(source.Name)) + continue + } + if sourceList != "" { sourceList += "," } @@ -102,7 +119,16 @@ func prepareProjectWithSources(ctx *builderContext) error { return fmt.Errorf("failure while writing the configuration application.properties: %w", err) } } - + if len(resourceList) > 0 { + if ctx.Build.Maven.Properties == nil { + ctx.Build.Maven.Properties = make(map[string]string) + } + resourceIncludes := strings.Join(resourceList, ",") + if existing := ctx.Build.Maven.Properties["quarkus.native.resources.includes"]; existing != "" { + resourceIncludes = existing + "," + resourceIncludes + } + ctx.Build.Maven.Properties["quarkus.native.resources.includes"] = resourceIncludes + } return nil } diff --git a/pkg/builder/quarkus_test.go b/pkg/builder/quarkus_test.go index 1fd8b9a4ed..7f19dd67cf 100644 --- a/pkg/builder/quarkus_test.go +++ b/pkg/builder/quarkus_test.go @@ -224,6 +224,46 @@ func TestGenerateQuarkusProjectWithNativeSources(t *testing.T) { require.NoError(t, err) } +func TestGenerateQuarkusProjectWithNativeResources(t *testing.T) { + tmpDir := t.TempDir() + defaultCatalog, err := camel.DefaultCatalog() + require.NoError(t, err) + + builderContext := builderContext{ + C: context.TODO(), + Path: tmpDir, + Namespace: "test", + Build: v1.BuilderTask{ + Runtime: defaultCatalog.Runtime, + Maven: v1.MavenBuildSpec{ + MavenSpec: v1.MavenSpec{}, + }, + Sources: []v1.SourceSpec{ + v1.NewSourceSpec("Test.java", "bogus, irrelevant for test", v1.LanguageJavaSource), + { + DataSpec: v1.DataSpec{ + Name: "resources/my-resource.yaml", + Content: "hello: from-resource", + }, + NativeImage: v1.NativeImageSourceTypeResource, + }, + }, + }, + } + + err = prepareProjectWithSources(&builderContext) + require.NoError(t, err) + + assert.Equal(t, "resources/my-resource.yaml", builderContext.Build.Maven.Properties["quarkus.native.resources.includes"]) + + materializedResource, err := os.ReadFile(filepath.Join(tmpDir, "maven", "src", "main", "resources", "resources", "my-resource.yaml")) + require.NoError(t, err) + assert.Equal(t, "hello: from-resource", string(materializedResource)) + + _, err = os.Stat(filepath.Join(tmpDir, "maven", "src", "main", "resources", "routes", "resources", "my-resource.yaml")) + assert.Error(t, err) +} + func TestBuildQuarkusRunner(t *testing.T) { tmpDir := t.TempDir() defaultCatalog, err := camel.DefaultCatalog() diff --git a/pkg/client/camel/applyconfiguration/camel/v1/sourcespec.go b/pkg/client/camel/applyconfiguration/camel/v1/sourcespec.go index ed3ee00c1f..93930495b3 100644 --- a/pkg/client/camel/applyconfiguration/camel/v1/sourcespec.go +++ b/pkg/client/camel/applyconfiguration/camel/v1/sourcespec.go @@ -32,6 +32,8 @@ type SourceSpecApplyConfiguration struct { DataSpecApplyConfiguration `json:",inline"` // specify which is the language (Camel DSL) used to interpret this source code Language *camelv1.Language `json:"language,omitempty"` + // Controls how the source is included when building a native executable. + NativeImage *camelv1.NativeImageSourceType `json:"nativeImage,omitempty"` // Loader is an optional id of the org.apache.camel.k.RoutesLoader that will // interpret this source at runtime Loader *string `json:"loader,omitempty"` @@ -128,6 +130,14 @@ func (b *SourceSpecApplyConfiguration) WithLanguage(value camelv1.Language) *Sou return b } +// WithNativeImage sets the NativeImage field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the NativeImage field is set to the value of the last call. +func (b *SourceSpecApplyConfiguration) WithNativeImage(value camelv1.NativeImageSourceType) *SourceSpecApplyConfiguration { + b.NativeImage = &value + return b +} + // WithLoader sets the Loader field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the Loader field is set to the value of the last call. diff --git a/pkg/controller/integration/kits.go b/pkg/controller/integration/kits.go index 5dd9c929fc..6dcc9cb221 100644 --- a/pkg/controller/integration/kits.go +++ b/pkg/controller/integration/kits.go @@ -209,13 +209,15 @@ func kitMatches(c client.Client, kit *v1.IntegrationKit, target *v1.IntegrationK } func hasMatchingSourcesForNative(it *v1.Integration, kit *v1.IntegrationKit) bool { - if len(it.OriginalSources()) != len(kit.Spec.Sources) { + if len(kit.Spec.Sources) == 0 { return false } - for _, itSource := range it.OriginalSources() { + for _, ikSource := range kit.Spec.Sources { found := false - for _, ikSource := range kit.Spec.Sources { - if itSource.Content == ikSource.Content { + for _, itSource := range it.OriginalSources() { + if itSource.Name == ikSource.Name && + itSource.Content == ikSource.Content && + itSource.NativeImageSourceType() == ikSource.NativeImageSourceType() { found = true break diff --git a/pkg/controller/integration/kits_test.go b/pkg/controller/integration/kits_test.go index c7f44b43cc..8a5a0d3d4a 100644 --- a/pkg/controller/integration/kits_test.go +++ b/pkg/controller/integration/kits_test.go @@ -384,6 +384,78 @@ func TestHasMatchingMultipleSources(t *testing.T) { assert.False(t, hms2) } +func TestHasMatchingSourcesAllowsNonBuildTimeIntegrationSources(t *testing.T) { + integration := &v1.Integration{ + Spec: v1.IntegrationSpec{ + Sources: []v1.SourceSpec{ + v1.NewSourceSpec("Test.java", "some java content", v1.LanguageJavaSource), + { + DataSpec: v1.DataSpec{ + Name: "routes.yaml", + Content: "some yaml route content", + }, + }, + { + DataSpec: v1.DataSpec{ + Name: "resources/my-resource.txt", + Content: "some resource content", + }, + NativeImage: v1.NativeImageSourceTypeResource, + }, + }, + }, + } + + kit := &v1.IntegrationKit{ + Spec: v1.IntegrationKitSpec{ + Sources: []v1.SourceSpec{ + v1.NewSourceSpec("Test.java", "some java content", v1.LanguageJavaSource), + { + DataSpec: v1.DataSpec{ + Name: "resources/my-resource.txt", + Content: "some resource content", + }, + NativeImage: v1.NativeImageSourceTypeResource, + }, + }, + }, + } + + assert.True(t, hasMatchingSourcesForNative(integration, kit)) +} + +func TestHasNotMatchingSourcesWhenNativeImageTypeDiffers(t *testing.T) { + integration := &v1.Integration{ + Spec: v1.IntegrationSpec{ + Sources: []v1.SourceSpec{ + { + DataSpec: v1.DataSpec{ + Name: "resource.yaml", + Content: "content", + }, + NativeImage: v1.NativeImageSourceTypeResource, + }, + }, + }, + } + + kit := &v1.IntegrationKit{ + Spec: v1.IntegrationKitSpec{ + Sources: []v1.SourceSpec{ + { + DataSpec: v1.DataSpec{ + Name: "resource.yaml", + Content: "content", + }, + NativeImage: v1.NativeImageSourceTypeRoute, + }, + }, + }, + } + + assert.False(t, hasMatchingSourcesForNative(integration, kit)) +} + func TestHasNotMatchingSources(t *testing.T) { integration := &v1.Integration{ Spec: v1.IntegrationSpec{ diff --git a/pkg/trait/quarkus.go b/pkg/trait/quarkus.go index bf8ca05e0c..5c28a94502 100644 --- a/pkg/trait/quarkus.go +++ b/pkg/trait/quarkus.go @@ -203,6 +203,9 @@ func (t *quarkusTrait) languageSettingDeprecated(e *Environment) bool { return false } for _, source := range e.Integration.AllSources() { + if source.NativeImageAsResource() { + continue + } if language := source.InferLanguage(); getLanguageSettings(e, language).deprecated { return true } @@ -213,6 +216,9 @@ func (t *quarkusTrait) languageSettingDeprecated(e *Environment) bool { func (t *quarkusTrait) validateNativeSupport(e *Environment) error { for _, source := range e.Integration.AllSources() { + if source.NativeImageAsResource() { + continue + } if language := source.InferLanguage(); !getLanguageSettings(e, language).native { return fmt.Errorf("invalid native support: Integration %s/%s contains a %s source that cannot be compiled to native executable", e.Integration.Namespace, e.Integration.Name, language) @@ -505,11 +511,15 @@ func sourcesRequiredAtBuildTime(e *Environment, source v1.SourceSpec) bool { return settings.native && settings.sourcesRequiredAtBuildTime } +func sourceMustBeEmbeddedAsResource(source v1.SourceSpec) bool { + return source.NativeImageAsResource() +} + // Propagates the user defined sources that are required at build time for native compilation. func propagateSourcesRequiredAtBuildTime(e *Environment) []v1.SourceSpec { array := make([]v1.SourceSpec, 0) for _, source := range e.Integration.OriginalSources() { - if sourcesRequiredAtBuildTime(e, source) { + if sourcesRequiredAtBuildTime(e, source) || sourceMustBeEmbeddedAsResource(source) { array = append(array, source) } } diff --git a/pkg/trait/quarkus_test.go b/pkg/trait/quarkus_test.go index d6b744bc77..a51fa6f6b4 100644 --- a/pkg/trait/quarkus_test.go +++ b/pkg/trait/quarkus_test.go @@ -72,6 +72,25 @@ func TestConfigureQuarkusTraitNativeNotSupported(t *testing.T) { assert.Nil(t, condition) } +func TestConfigureQuarkusTraitNativeResourceSkipsValidation(t *testing.T) { + quarkusTrait, environment := createNominalQuarkusTest() + environment.Integration.Spec.Sources[0] = v1.SourceSpec{ + DataSpec: v1.DataSpec{ + Name: "not-a-route.js", + Content: "plain text resource", + }, + NativeImage: v1.NativeImageSourceTypeResource, + } + environment.Integration.Status.Phase = v1.IntegrationPhaseBuildingKit + quarkusTrait.Modes = []traitv1.QuarkusMode{traitv1.NativeQuarkusMode} + + configured, condition, err := quarkusTrait.Configure(environment) + + assert.True(t, configured) + require.NoError(t, err) + assert.Nil(t, condition) +} + func TestApplyQuarkusTraitDefaultKitLayout(t *testing.T) { quarkusTrait, environment := createNominalQuarkusTest() environment.Integration.Status.Phase = v1.IntegrationPhaseBuildingKit diff --git a/pkg/trait/resolver.go b/pkg/trait/resolver.go index d2ab9feef0..6a1363f9fa 100644 --- a/pkg/trait/resolver.go +++ b/pkg/trait/resolver.go @@ -99,8 +99,15 @@ func resolveIntegrationSources( } else { sources = integration.AllSources() } + filtered := make([]v1.SourceSpec, 0, len(sources)) + for _, source := range sources { + if source.NativeImageAsResource() { + continue + } + filtered = append(filtered, source) + } - return resolveSources(sources, func(name string) (*corev1.ConfigMap, error) { + return resolveSources(filtered, func(name string) (*corev1.ConfigMap, error) { // the config map could be part of the resources created // by traits cm := resources.GetConfigMap(func(m *corev1.ConfigMap) bool { diff --git a/pkg/util/digest/digest.go b/pkg/util/digest/digest.go index 7849642597..630e75e1e2 100644 --- a/pkg/util/digest/digest.go +++ b/pkg/util/digest/digest.go @@ -279,6 +279,9 @@ func ComputeForSource(s v1.SourceSpec) (string, error) { if _, err := hash.Write([]byte(s.Language)); err != nil { return "", err } + if _, err := hash.Write([]byte(s.NativeImageSourceType())); err != nil { + return "", err + } if _, err := hash.Write([]byte(s.ContentKey)); err != nil { return "", err }