diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml index fa1b42930a..9015ee97f7 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml @@ -668,6 +668,14 @@ spec: with the IAM role for the instance. The instance profile contains the IAM role. type: string + id: + description: |- + ID is the ID of an existing launch template (e.g. lt-xxxx). When set, CAPA will + not create or manage the launch template and will use the referenced one directly. + type: string + x-kubernetes-validations: + - message: ID is immutable + rule: self == oldSelf imageLookupBaseOS: description: |- ImageLookupBaseOS is the name of the base operating system to use for @@ -784,6 +792,9 @@ spec: name: description: The name of the launch template. type: string + x-kubernetes-validations: + - message: Name is immutable once set + rule: self == oldSelf nonRootVolumes: description: Configuration options for the non root storage volumes. items: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml index 20962dec31..3b7a05b616 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml @@ -591,6 +591,8 @@ spec: AWSLaunchTemplate specifies the launch template to use to create the managed node group. If AWSLaunchTemplate is specified, certain node group configuraions outside of launch template are prohibited (https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html). + When AWSLaunchTemplate.ID is set, CAPA treats the template as BYO and does not create or + delete the launch template. properties: additionalSecurityGroups: description: |- @@ -677,6 +679,14 @@ spec: with the IAM role for the instance. The instance profile contains the IAM role. type: string + id: + description: |- + ID is the ID of an existing launch template (e.g. lt-xxxx). When set, CAPA will + not create or manage the launch template and will use the referenced one directly. + type: string + x-kubernetes-validations: + - message: ID is immutable + rule: self == oldSelf imageLookupBaseOS: description: |- ImageLookupBaseOS is the name of the base operating system to use for @@ -793,6 +803,9 @@ spec: name: description: The name of the launch template. type: string + x-kubernetes-validations: + - message: Name is immutable once set + rule: self == oldSelf nonRootVolumes: description: Configuration options for the non root storage volumes. items: diff --git a/exp/api/v1beta1/conversion.go b/exp/api/v1beta1/conversion.go index 7b49ed9f64..36657585ad 100644 --- a/exp/api/v1beta1/conversion.go +++ b/exp/api/v1beta1/conversion.go @@ -47,6 +47,10 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error { if restored.Spec.AWSLaunchTemplate.InstanceMetadataOptions != nil { dst.Spec.AWSLaunchTemplate.InstanceMetadataOptions = restored.Spec.AWSLaunchTemplate.InstanceMetadataOptions } + // ID is a v1beta2-only field; restore it from the annotation. + if restored.Spec.AWSLaunchTemplate.ID != nil { + dst.Spec.AWSLaunchTemplate.ID = restored.Spec.AWSLaunchTemplate.ID + } if restored.Spec.AvailabilityZoneSubnetType != nil { dst.Spec.AvailabilityZoneSubnetType = restored.Spec.AvailabilityZoneSubnetType } @@ -118,6 +122,10 @@ func (src *AWSManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { if dst.Spec.AWSLaunchTemplate == nil { dst.Spec.AWSLaunchTemplate = restored.Spec.AWSLaunchTemplate } + // ID is a v1beta2-only field (BYO launch template); restore it from the annotation. + if restored.Spec.AWSLaunchTemplate.ID != nil { + dst.Spec.AWSLaunchTemplate.ID = restored.Spec.AWSLaunchTemplate.ID + } dst.Spec.AWSLaunchTemplate.InstanceMetadataOptions = restored.Spec.AWSLaunchTemplate.InstanceMetadataOptions dst.Spec.AWSLaunchTemplate.NonRootVolumes = restored.Spec.AWSLaunchTemplate.NonRootVolumes @@ -233,6 +241,8 @@ func (r *AWSFargateProfileList) ConvertFrom(srcRaw conversion.Hub) error { } // Convert_v1beta2_AWSLaunchTemplate_To_v1beta1_AWSLaunchTemplate converts the v1beta2 AWSLaunchTemplate receiver to a v1beta1 AWSLaunchTemplate. +// AWSLaunchTemplate.ID (BYO launch template) is a v1beta2-only field and is intentionally dropped on downgrade; +// v1beta1 is deprecated and will not gain new fields. func Convert_v1beta2_AWSLaunchTemplate_To_v1beta1_AWSLaunchTemplate(in *expinfrav1.AWSLaunchTemplate, out *AWSLaunchTemplate, s apiconversion.Scope) error { return autoConvert_v1beta2_AWSLaunchTemplate_To_v1beta1_AWSLaunchTemplate(in, out, s) } diff --git a/exp/api/v1beta1/zz_generated.conversion.go b/exp/api/v1beta1/zz_generated.conversion.go index e825e20a31..7fdea6a8dc 100644 --- a/exp/api/v1beta1/zz_generated.conversion.go +++ b/exp/api/v1beta1/zz_generated.conversion.go @@ -397,6 +397,7 @@ func Convert_v1beta1_AWSLaunchTemplate_To_v1beta2_AWSLaunchTemplate(in *AWSLaunc } func autoConvert_v1beta2_AWSLaunchTemplate_To_v1beta1_AWSLaunchTemplate(in *v1beta2.AWSLaunchTemplate, out *AWSLaunchTemplate, s conversion.Scope) error { + // WARNING: in.ID requires manual conversion: does not exist in peer-type out.Name = in.Name out.IamInstanceProfile = in.IamInstanceProfile out.AMI = in.AMI diff --git a/exp/api/v1beta2/awsmanagedmachinepool_types.go b/exp/api/v1beta2/awsmanagedmachinepool_types.go index 8e761a506f..0437e427ba 100644 --- a/exp/api/v1beta2/awsmanagedmachinepool_types.go +++ b/exp/api/v1beta2/awsmanagedmachinepool_types.go @@ -208,6 +208,8 @@ type AWSManagedMachinePoolSpec struct { // AWSLaunchTemplate specifies the launch template to use to create the managed node group. // If AWSLaunchTemplate is specified, certain node group configuraions outside of launch template // are prohibited (https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html). + // When AWSLaunchTemplate.ID is set, CAPA treats the template as BYO and does not create or + // delete the launch template. // +optional AWSLaunchTemplate *AWSLaunchTemplate `json:"awsLaunchTemplate,omitempty"` diff --git a/exp/api/v1beta2/types.go b/exp/api/v1beta2/types.go index b4eca931a8..eb0e9a4a9b 100644 --- a/exp/api/v1beta2/types.go +++ b/exp/api/v1beta2/types.go @@ -61,7 +61,16 @@ type BlockDeviceMapping struct { // AWSLaunchTemplate defines the desired state of AWSLaunchTemplate. type AWSLaunchTemplate struct { + // ID is the ID of an existing launch template (e.g. lt-xxxx). When set, CAPA will + // not create or manage the launch template and will use the referenced one directly. + // +optional + // +immutable + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ID is immutable" + ID *string `json:"id,omitempty"` + // The name of the launch template. + // +immutable + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Name is immutable once set" Name string `json:"name,omitempty"` // The name or the Amazon Resource Name (ARN) of the instance profile associated diff --git a/exp/api/v1beta2/zz_generated.deepcopy.go b/exp/api/v1beta2/zz_generated.deepcopy.go index 9f623c1803..d0e44f9651 100644 --- a/exp/api/v1beta2/zz_generated.deepcopy.go +++ b/exp/api/v1beta2/zz_generated.deepcopy.go @@ -92,6 +92,11 @@ func (in *AWSFargateProfileList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AWSLaunchTemplate) DeepCopyInto(out *AWSLaunchTemplate) { *out = *in + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(string) + **out = **in + } in.AMI.DeepCopyInto(&out.AMI) if in.RootVolume != nil { in, out := &in.RootVolume, &out.RootVolume diff --git a/exp/controllers/awsmanagedmachinepool_controller.go b/exp/controllers/awsmanagedmachinepool_controller.go index efd66cc5e0..59d2a87e98 100644 --- a/exp/controllers/awsmanagedmachinepool_controller.go +++ b/exp/controllers/awsmanagedmachinepool_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "strconv" "time" autoscalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" @@ -212,7 +213,15 @@ func (r *AWSManagedMachinePoolReconciler) reconcileNormal( ec2svc := r.getEC2Service(ec2Scope) reconSvc := r.getReconcileService(ec2Scope) - if machinePoolScope.ManagedMachinePool.Spec.AWSLaunchTemplate != nil { + // BYO launch template: use existing LT; do not create, update, or delete it. + if machinePoolScope.IsBYOLaunchTemplate() { + lt := machinePoolScope.ManagedMachinePool.Spec.AWSLaunchTemplate + machinePoolScope.SetLaunchTemplateIDStatus(*lt.ID) + if lt.VersionNumber != nil { + machinePoolScope.SetLaunchTemplateLatestVersionStatus(strconv.FormatInt(*lt.VersionNumber, 10)) + } + v1beta1conditions.MarkTrue(machinePoolScope.ManagedMachinePool, expinfrav1.LaunchTemplateReadyCondition) + } else if machinePoolScope.ManagedMachinePool.Spec.AWSLaunchTemplate != nil { canStartInstanceRefresh := func() (bool, *autoscalingtypes.InstanceRefreshStatus, error) { return true, nil, nil } @@ -268,7 +277,8 @@ func (r *AWSManagedMachinePoolReconciler) reconcileDelete( return errors.Wrapf(err, "failed to reconcile machine pool deletion for AWSManagedMachinePool %s/%s", machinePoolScope.ManagedMachinePool.Namespace, machinePoolScope.ManagedMachinePool.Name) } - if machinePoolScope.ManagedMachinePool.Spec.AWSLaunchTemplate != nil { + // Only delete the launch template when it is CAPA-managed (not BYO). + if machinePoolScope.ManagedMachinePool.Spec.AWSLaunchTemplate != nil && !machinePoolScope.IsBYOLaunchTemplate() { launchTemplateID := machinePoolScope.ManagedMachinePool.Status.LaunchTemplateID launchTemplate, _, _, _, err := ec2Svc.GetLaunchTemplate(machinePoolScope.LaunchTemplateName()) if err != nil { diff --git a/exp/webhooks/awsmanagedmachinepool_webhook.go b/exp/webhooks/awsmanagedmachinepool_webhook.go index f4ce36ebef..c77b3f2f37 100644 --- a/exp/webhooks/awsmanagedmachinepool_webhook.go +++ b/exp/webhooks/awsmanagedmachinepool_webhook.go @@ -131,15 +131,69 @@ func (w *AWSManagedMachinePool) validateLaunchTemplate(r *expinfrav1.AWSManagedM return allErrs } - if r.Spec.InstanceType != nil { - allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "InstanceType"), r.Spec.InstanceType, "InstanceType cannot be specified when LaunchTemplate is specified")) + lt := r.Spec.AWSLaunchTemplate + ltPath := field.NewPath("spec", "awsLaunchTemplate") + isBYO := lt.ID != nil && *lt.ID != "" + + // For CAPA-managed LTs (no id), spec.instanceType is forbidden because the instance type + // must be configured inside the launch template itself. For BYO LTs (id is set), the AWS + // CreateNodegroup API allows InstanceTypes to be specified alongside the launch template + // when the launch template itself does not specify an instance type. + if r.Spec.InstanceType != nil && !isBYO { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "instanceType"), r.Spec.InstanceType, "instanceType cannot be specified with a CAPA-managed launch template; set spec.awsLaunchTemplate.instanceType instead")) } if r.Spec.DiskSize != nil { - allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "DiskSize"), r.Spec.DiskSize, "DiskSize cannot be specified when LaunchTemplate is specified")) + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "diskSize"), r.Spec.DiskSize, "diskSize cannot be specified when LaunchTemplate is specified")) + } + // remoteAccess is rejected by the EKS CreateNodegroup API whenever a launch template + // is specified, regardless of the template contents. SSH key and source security groups + // must be configured inside the launch template instead. + if r.Spec.RemoteAccess != nil { + allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "remoteAccess"), "remoteAccess cannot be specified when a launch template is specified; configure KeyName and security groups in the launch template instead")) + } + + if lt.IamInstanceProfile != "" { + allErrs = append(allErrs, field.Invalid(ltPath.Child("iamInstanceProfile"), lt.IamInstanceProfile, "IAM instance profile in launch template is prohibited in EKS managed node group")) } - if r.Spec.AWSLaunchTemplate.IamInstanceProfile != "" { - allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "AWSLaunchTemplate", "IamInstanceProfile"), r.Spec.AWSLaunchTemplate.IamInstanceProfile, "IAM instance profile in launch template is prohibited in EKS managed node group")) + // When using a BYO launch template (ID is set), versionNumber is required and + // CAPA-managed fields must not be specified. + if isBYO { + if lt.VersionNumber == nil { + allErrs = append(allErrs, field.Required(ltPath.Child("versionNumber"), "versionNumber is required when using a BYO launch template (id is set)")) + } + if lt.AMI.ID != nil || lt.AMI.EKSOptimizedLookupType != nil { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("ami"), "ami cannot be specified with a BYO launch template (id is set)")) + } + if lt.InstanceType != "" { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("instanceType"), "instanceType cannot be specified with a BYO launch template (id is set)")) + } + if lt.RootVolume != nil { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("rootVolume"), "rootVolume cannot be specified with a BYO launch template (id is set)")) + } + if len(lt.NonRootVolumes) > 0 { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("nonRootVolumes"), "nonRootVolumes cannot be specified with a BYO launch template (id is set)")) + } + if lt.SSHKeyName != nil { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("sshKeyName"), "sshKeyName cannot be specified with a BYO launch template (id is set)")) + } + if lt.ImageLookupFormat != "" { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("imageLookupFormat"), "imageLookupFormat cannot be specified with a BYO launch template (id is set)")) + } + if lt.ImageLookupOrg != "" { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("imageLookupOrg"), "imageLookupOrg cannot be specified with a BYO launch template (id is set)")) + } + if lt.ImageLookupBaseOS != "" { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("imageLookupBaseOS"), "imageLookupBaseOS cannot be specified with a BYO launch template (id is set)")) + } + if len(lt.AdditionalSecurityGroups) > 0 { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("additionalSecurityGroups"), "additionalSecurityGroups cannot be specified with a BYO launch template (id is set)")) + } + // spec.amiType and spec.amiVersion are intentionally allowed alongside a BYO launch + // template: the AWS CreateNodegroup API accepts them as long as the launch template + // does not pin a custom AMI. CAPA cannot introspect user-owned launch templates, so + // any conflict between these fields and the referenced template is surfaced by the + // EKS API at create time rather than at admission. } return allErrs @@ -278,8 +332,31 @@ func (w *AWSManagedMachinePool) validateImmutable(r *expinfrav1.AWSManagedMachin field.Invalid(field.NewPath("spec", "AWSLaunchTemplate"), old.Spec.AWSLaunchTemplate, "field is immutable"), ) } - if old.Spec.AWSLaunchTemplate != nil && r.Spec.AWSLaunchTemplate != nil { - appendErrorIfMutated(old.Spec.AWSLaunchTemplate.Name, r.Spec.AWSLaunchTemplate.Name, "awsLaunchTemplate.name") + allErrs = append(allErrs, w.validateLaunchTemplateImmutability(r, old)...) + + return allErrs +} + +// validateLaunchTemplateImmutability ensures that immutable fields within AWSLaunchTemplate +// (ID and Name) are not modified after creation. VersionNumber is intentionally excluded +// as it may be updated to roll out a new launch template version to the nodegroup. +func (w *AWSManagedMachinePool) validateLaunchTemplateImmutability(r *expinfrav1.AWSManagedMachinePool, old *expinfrav1.AWSManagedMachinePool) field.ErrorList { + var allErrs field.ErrorList + + oldLT := old.Spec.AWSLaunchTemplate + newLT := r.Spec.AWSLaunchTemplate + + if oldLT == nil || newLT == nil { + return allErrs + } + + ltPath := field.NewPath("spec", "awsLaunchTemplate") + + if !reflect.DeepEqual(oldLT.ID, newLT.ID) { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("id"), "id is immutable")) + } + if !reflect.DeepEqual(oldLT.Name, newLT.Name) { + allErrs = append(allErrs, field.Forbidden(ltPath.Child("name"), "name is immutable")) } return allErrs diff --git a/exp/webhooks/awsmanagedmachinepool_webhook_test.go b/exp/webhooks/awsmanagedmachinepool_webhook_test.go index 61580b38f2..06e311f438 100644 --- a/exp/webhooks/awsmanagedmachinepool_webhook_test.go +++ b/exp/webhooks/awsmanagedmachinepool_webhook_test.go @@ -156,6 +156,158 @@ func TestAWSManagedMachinePoolValidateCreate(t *testing.T) { }, wantErr: false, }, + // BYO launch template tests (AWSLaunchTemplate with ID set) + { + name: "valid BYO launch template with ID", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-byo", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + wantErr: false, + }, + { + name: "BYO launch template without versionNumber is rejected", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-byo", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + }, + }, + }, + wantErr: true, + }, + { + name: "BYO launch template with diskSize is rejected", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-byo", + DiskSize: &oldDiskSize, + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + wantErr: true, + }, + { + name: "BYO launch template with spec.instanceType is accepted", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-byo", + InstanceType: aws.String("m5.large"), + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + wantErr: false, + }, + { + name: "CAPA-managed launch template with spec.instanceType is rejected", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group", + InstanceType: aws.String("m5.large"), + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + Name: "my-lt", + }, + }, + }, + wantErr: true, + }, + { + name: "BYO launch template with spec.amiType is accepted", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-byo", + AMIType: ptr.To(expinfrav1.Al2023x86_64), + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + wantErr: false, + }, + { + name: "BYO launch template with spec.amiVersion is accepted", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-byo", + AMIVersion: aws.String("1.29.0-20240110"), + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + wantErr: false, + }, + { + name: "BYO launch template with spec.remoteAccess is rejected", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-byo", + RemoteAccess: &expinfrav1.ManagedRemoteAccess{ + Public: true, + }, + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + wantErr: true, + }, + { + name: "CAPA-managed launch template with spec.remoteAccess is rejected", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group", + RemoteAccess: &expinfrav1.ManagedRemoteAccess{ + Public: true, + }, + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + Name: "my-lt", + }, + }, + }, + wantErr: true, + }, + { + name: "BYO launch template with AMI is rejected", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-byo", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + AMI: infrav1.AMIReference{ID: aws.String("ami-123")}, + }, + }, + }, + wantErr: true, + }, + { + name: "BYO launch template with instanceType in LT is rejected", + pool: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-byo", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + InstanceType: "m5.large", + }, + }, + }, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -694,6 +846,94 @@ func TestAWSManagedMachinePoolValidateUpdate(t *testing.T) { }, wantErr: false, }, + // BYO launch template (AWSLaunchTemplate.ID) immutability tests + { + name: "changing BYO launch template ID is rejected", + old: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-1", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + new: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-1", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-99999"), + VersionNumber: aws.Int64(1), + }, + }, + }, + wantErr: true, + }, + { + name: "setting BYO launch template ID on CAPA-managed LT is rejected", + old: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-1", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + Name: "my-lt", + }, + }, + }, + new: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-1", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + Name: "my-lt", + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + wantErr: true, + }, + { + name: "removing BYO launch template ID is rejected", + old: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-1", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + new: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-1", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + Name: "my-lt", + }, + }, + }, + wantErr: true, + }, + { + name: "changing BYO launch template version is accepted", + old: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-1", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(1), + }, + }, + }, + new: &expinfrav1.AWSManagedMachinePool{ + Spec: expinfrav1.AWSManagedMachinePoolSpec{ + EKSNodegroupName: "eks-node-group-1", + AWSLaunchTemplate: &expinfrav1.AWSLaunchTemplate{ + ID: aws.String("lt-12345"), + VersionNumber: aws.Int64(3), + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/cloud/scope/managednodegroup.go b/pkg/cloud/scope/managednodegroup.go index d0b2116706..9c1f5db392 100644 --- a/pkg/cloud/scope/managednodegroup.go +++ b/pkg/cloud/scope/managednodegroup.go @@ -413,13 +413,26 @@ func (s *ManagedMachinePoolScope) GetLaunchTemplate() *expinfrav1.AWSLaunchTempl return s.ManagedMachinePool.Spec.AWSLaunchTemplate } +// IsBYOLaunchTemplate returns true when the pool uses an existing (user-provided) launch template. +func (s *ManagedMachinePoolScope) IsBYOLaunchTemplate() bool { + lt := s.ManagedMachinePool.Spec.AWSLaunchTemplate + return lt != nil && lt.ID != nil && *lt.ID != "" +} + // GetMachinePool returns the machine pool. func (s *ManagedMachinePoolScope) GetMachinePool() *clusterv1.MachinePool { return s.MachinePool } // LaunchTemplateName returns the launch template name. +// When using a BYO launch template with a user-provided name, it returns that name +// to prevent drift between user-provided and derived names. func (s *ManagedMachinePoolScope) LaunchTemplateName() string { + if s.IsBYOLaunchTemplate() { + if name := s.ManagedMachinePool.Spec.AWSLaunchTemplate.Name; name != "" { + return name + } + } return fmt.Sprintf("%s-%s", s.ControlPlane.Name, s.ManagedMachinePool.Name) } diff --git a/pkg/cloud/services/eks/nodegroup.go b/pkg/cloud/services/eks/nodegroup.go index e89c97b16a..3b5cd75a47 100644 --- a/pkg/cloud/services/eks/nodegroup.go +++ b/pkg/cloud/services/eks/nodegroup.go @@ -222,16 +222,32 @@ func (s *NodegroupService) createNodegroup(ctx context.Context) (*ekstypes.Nodeg NodeRole: roleArn, Labels: managedPool.Labels, Tags: tags, - RemoteAccess: remoteAccess, UpdateConfig: updatedConfig, } - if managedPool.AMIType != nil && (managedPool.AWSLaunchTemplate == nil || managedPool.AWSLaunchTemplate.AMI.ID == nil) { - input.AmiType = converters.AMITypeToSDK(*managedPool.AMIType) + useLaunchTemplate := managedPool.AWSLaunchTemplate != nil + isBYO := useLaunchTemplate && managedPool.AWSLaunchTemplate.ID != nil && *managedPool.AWSLaunchTemplate.ID != "" + // RemoteAccess is rejected by EKS when a launch template is specified; key pair and + // source security groups must live in the launch template. Webhook validation also + // enforces this, but guard the service layer for defence in depth. + if !useLaunchTemplate { + input.RemoteAccess = remoteAccess + } + // AmiType can be passed alongside a launch template as long as the template does not + // pin a custom AMI. For CAPA-managed LTs, a custom AMI is indicated by LT.AMI.ID. + // For BYO LTs we cannot introspect the referenced template, so the value flows through + // and AWS rejects it if the template already specifies an AMI type (custom AMI). + if managedPool.AMIType != nil { + ltHasCustomAMI := useLaunchTemplate && !isBYO && managedPool.AWSLaunchTemplate.AMI.ID != nil + if !ltHasCustomAMI { + input.AmiType = converters.AMITypeToSDK(*managedPool.AMIType) + } } - if managedPool.DiskSize != nil { + if managedPool.DiskSize != nil && !useLaunchTemplate { input.DiskSize = managedPool.DiskSize } - if managedPool.InstanceType != nil { + // InstanceTypes may be specified without a launch template, or alongside a BYO launch + // template when the launch template itself does not specify an instance type. + if managedPool.InstanceType != nil && (!useLaunchTemplate || isBYO) { input.InstanceTypes = []string{aws.ToString(managedPool.InstanceType)} } if len(managedPool.Taints) > 0 { @@ -249,7 +265,7 @@ func (s *NodegroupService) createNodegroup(ctx context.Context) (*ekstypes.Nodeg } input.CapacityType = capacityType } - if managedPool.AWSLaunchTemplate != nil { + if useLaunchTemplate { input.LaunchTemplate = &ekstypes.LaunchTemplateSpecification{ Id: s.scope.ManagedMachinePool.Status.LaunchTemplateID, Version: s.scope.ManagedMachinePool.Status.LaunchTemplateVersion, @@ -317,6 +333,33 @@ func (s *NodegroupService) deleteNodegroupAndWait(ctx context.Context) (reterr e return nil } +// isSymbolicLaunchTemplateVersion returns true for AWS symbolic version aliases +// ("$Latest", "$Default") that resolve to a concrete version number at apply time. +// When comparing a symbolic spec version against the nodegroup's resolved version, +// we skip the comparison to avoid an endless update loop. +func isSymbolicLaunchTemplateVersion(v string) bool { + return v == "$Latest" || v == "$Default" +} + +// launchTemplateNeedsUpdate returns true when either the launch template ID or +// version in the desired status differs from what the nodegroup currently uses. +// +// - statusID / statusVersion are the values CAPA wants (from ManagedMachinePool.Status). +// - ngID / ngVersion are what the live nodegroup currently has (may be nil when no LT is attached). +func launchTemplateNeedsUpdate(statusID, statusVersion, ngID, ngVersion *string) bool { + // ID changed: status has an ID and either the nodegroup has none, or a different one. + idChanged := statusID != nil && (ngID == nil || *statusID != *ngID) + + // Version changed: both sides have a version, the desired version is concrete + // (not a symbolic alias like "$Latest"), and the values differ. + versionChanged := statusVersion != nil && + ngVersion != nil && + !isSymbolicLaunchTemplateVersion(*statusVersion) && + *statusVersion != *ngVersion + + return idChanged || versionChanged +} + func (s *NodegroupService) reconcileNodegroupVersion(ctx context.Context, ng *ekstypes.Nodegroup) error { var specVersion *version.Version if s.scope.Version() != nil { @@ -334,28 +377,34 @@ func (s *NodegroupService) reconcileNodegroupVersion(ctx context.Context, ng *ek ngVersion := version.MustParseGeneric(*ng.Version) specAMI := s.scope.ManagedMachinePool.Spec.AMIVersion ngAMI := *ng.ReleaseVersion + + statusLaunchTemplateID := s.scope.ManagedMachinePool.Status.LaunchTemplateID statusLaunchTemplateVersion := s.scope.ManagedMachinePool.Status.LaunchTemplateVersion + var ngLaunchTemplateID *string var ngLaunchTemplateVersion *string if ng.LaunchTemplate != nil { + ngLaunchTemplateID = ng.LaunchTemplate.Id ngLaunchTemplateVersion = ng.LaunchTemplate.Version } + launchTemplateChanged := launchTemplateNeedsUpdate(statusLaunchTemplateID, statusLaunchTemplateVersion, ngLaunchTemplateID, ngLaunchTemplateVersion) + eksClusterName := s.scope.KubernetesClusterName() - if (specVersion != nil && ngVersion.LessThan(specVersion)) || (specAMI != nil && *specAMI != ngAMI) || (statusLaunchTemplateVersion != nil && *statusLaunchTemplateVersion != *ngLaunchTemplateVersion) { + if (specVersion != nil && ngVersion.LessThan(specVersion)) || (specAMI != nil && *specAMI != ngAMI) || launchTemplateChanged { input := &eks.UpdateNodegroupVersionInput{ ClusterName: aws.String(eksClusterName), NodegroupName: aws.String(s.scope.NodegroupName()), } var updateMsg string - // Either update k8s version or AMI version + // Either update launch template, k8s version, or AMI version (in that priority order) switch { - case statusLaunchTemplateVersion != nil && *statusLaunchTemplateVersion != *ngLaunchTemplateVersion: + case launchTemplateChanged: input.LaunchTemplate = &ekstypes.LaunchTemplateSpecification{ - Id: s.scope.ManagedMachinePool.Status.LaunchTemplateID, + Id: statusLaunchTemplateID, Version: statusLaunchTemplateVersion, } - updateMsg = fmt.Sprintf("to launch template version %s", *statusLaunchTemplateVersion) + updateMsg = fmt.Sprintf("to launch template %s version %s", aws.ToString(statusLaunchTemplateID), aws.ToString(statusLaunchTemplateVersion)) case specVersion != nil && ngVersion.LessThan(specVersion): // NOTE: you can only upgrade increments of minor versions. If you want to upgrade 1.14 to 1.16 we // need to go 1.14-> 1.15 and then 1.15 -> 1.16. @@ -535,6 +584,12 @@ func (s *NodegroupService) reconcileNodegroup(ctx context.Context) error { return errors.Wrap(err, "failed to set status") } + // A failed nodegroup has nil Version/ReleaseVersion and cannot be updated; + // return early so the FailureMessage set above surfaces to the user. + if ng.Status == ekstypes.NodegroupStatusCreateFailed || ng.Status == ekstypes.NodegroupStatusDeleteFailed { + return nil + } + switch ng.Status { case ekstypes.NodegroupStatusCreating, ekstypes.NodegroupStatusUpdating: ng, err = s.waitForNodegroupActive(ctx) diff --git a/pkg/cloud/services/eks/nodegroup_test.go b/pkg/cloud/services/eks/nodegroup_test.go new file mode 100644 index 0000000000..4997941755 --- /dev/null +++ b/pkg/cloud/services/eks/nodegroup_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eks + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" +) + +func TestIsSymbolicLaunchTemplateVersion(t *testing.T) { + tests := []struct { + name string + version string + want bool + }{ + {name: "$Latest is symbolic", version: "$Latest", want: true}, + {name: "$Default is symbolic", version: "$Default", want: true}, + {name: "concrete version 1", version: "1", want: false}, + {name: "concrete version 42", version: "42", want: false}, + {name: "empty string", version: "", want: false}, + {name: "lowercase $latest is not symbolic", version: "$latest", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isSymbolicLaunchTemplateVersion(tt.version); got != tt.want { + t.Errorf("isSymbolicLaunchTemplateVersion(%q) = %v, want %v", tt.version, got, tt.want) + } + }) + } +} + +func TestLaunchTemplateNeedsUpdate(t *testing.T) { + tests := []struct { + name string + statusID *string + statusVersion *string + ngID *string + ngVersion *string + want bool + }{ + { + name: "no launch template in status or nodegroup — no update", + want: false, + }, + { + name: "status has ID but nodegroup has no LT — update needed (BYO LT being applied for first time)", + statusID: aws.String("lt-12345"), + want: true, + }, + { + name: "same ID, no version — no update", + statusID: aws.String("lt-12345"), + ngID: aws.String("lt-12345"), + want: false, + }, + { + name: "ID changed — update needed", + statusID: aws.String("lt-99999"), + ngID: aws.String("lt-12345"), + want: true, + }, + { + name: "same ID, concrete version unchanged — no update", + statusID: aws.String("lt-12345"), + statusVersion: aws.String("3"), + ngID: aws.String("lt-12345"), + ngVersion: aws.String("3"), + want: false, + }, + { + name: "same ID, concrete version changed — update needed", + statusID: aws.String("lt-12345"), + statusVersion: aws.String("4"), + ngID: aws.String("lt-12345"), + ngVersion: aws.String("3"), + want: true, + }, + { + name: "same ID, status version is $Latest — no update (symbolic version skipped)", + statusID: aws.String("lt-12345"), + statusVersion: aws.String("$Latest"), + ngID: aws.String("lt-12345"), + ngVersion: aws.String("7"), + want: false, + }, + { + name: "same ID, status version is $Default — no update (symbolic version skipped)", + statusID: aws.String("lt-12345"), + statusVersion: aws.String("$Default"), + ngID: aws.String("lt-12345"), + ngVersion: aws.String("2"), + want: false, + }, + { + name: "status has version but no ID, nodegroup has neither — no update", + statusVersion: aws.String("5"), + want: false, + }, + { + name: "status has no version, nodegroup has version — no update", + statusID: aws.String("lt-12345"), + ngID: aws.String("lt-12345"), + ngVersion: aws.String("2"), + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := launchTemplateNeedsUpdate(tt.statusID, tt.statusVersion, tt.ngID, tt.ngVersion); got != tt.want { + t.Errorf("launchTemplateNeedsUpdate() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/test/e2e/data/e2e_eks_conf.yaml b/test/e2e/data/e2e_eks_conf.yaml index a5fc5863f5..379dfb4443 100644 --- a/test/e2e/data/e2e_eks_conf.yaml +++ b/test/e2e/data/e2e_eks_conf.yaml @@ -112,6 +112,8 @@ providers: targetName: "cluster-template-eks-machinepool-only.yaml" - sourcePath: "./eks/cluster-template-eks-managed-machinepool-with-launch-template-only.yaml" targetName: "cluster-template-eks-managed-machinepool-with-launch-template-only.yaml" + - sourcePath: "./eks/cluster-template-eks-managed-machinepool-with-byo-launch-template-only.yaml" + targetName: "cluster-template-eks-managed-machinepool-with-byo-launch-template-only.yaml" - sourcePath: "./eks/cluster-template-eks-managedmachinepool.yaml" targetName: "cluster-template-eks-managedmachinepool.yaml" - sourcePath: "./eks/cluster-template-eks-ipv6-cluster.yaml" diff --git a/test/e2e/data/eks/cluster-template-eks-managed-machinepool-with-byo-launch-template-only.yaml b/test/e2e/data/eks/cluster-template-eks-managed-machinepool-with-byo-launch-template-only.yaml new file mode 100644 index 0000000000..535c39cc2e --- /dev/null +++ b/test/e2e/data/eks/cluster-template-eks-managed-machinepool-with-byo-launch-template-only.yaml @@ -0,0 +1,32 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: "${CLUSTER_NAME}-pool-byo-lt-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + version: "${KUBERNETES_VERSION}" + clusterName: "${CLUSTER_NAME}" + bootstrap: + dataSecretName: "" + infrastructureRef: + name: "${CLUSTER_NAME}-pool-byo-lt-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSManagedMachinePool +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSManagedMachinePool +metadata: + name: "${CLUSTER_NAME}-pool-byo-lt-0" +spec: + eksNodegroupName: "${CLUSTER_NAME}-pool-byo-lt-0" + # instanceType must not be set when using a BYO launch template — EKS uses + # the type configured inside the launch template itself. + awsLaunchTemplate: + id: "${BYO_LAUNCH_TEMPLATE_ID}" + versionNumber: ${BYO_LAUNCH_TEMPLATE_VERSION} + scaling: + minSize: 1 + maxSize: 2 diff --git a/test/e2e/suites/managed/eks_test.go b/test/e2e/suites/managed/eks_test.go index c84e3c96e1..10d8b44a92 100644 --- a/test/e2e/suites/managed/eks_test.go +++ b/test/e2e/suites/managed/eks_test.go @@ -161,6 +161,24 @@ var _ = ginkgo.Describe("[managed] [general] EKS cluster tests", func() { } }) + ginkgo.By("should create a managed node pool with a BYO (user-provided) launch template") + byoLaunchTemplateName := fmt.Sprintf("%s-%s-byo-lt", namespace.Name, clusterName) + byoLT := CreateBYOLaunchTemplate(ctx, byoLaunchTemplateName, e2eCtx.BootstrapUserAWSSession) + defer DeleteBYOLaunchTemplate(ctx, byoLT.ID, e2eCtx.BootstrapUserAWSSession) + + BYOMachinePoolSpec(ctx, func() BYOMachinePoolSpecInput { + return BYOMachinePoolSpecInput{ + E2EConfig: e2eCtx.E2EConfig, + ConfigClusterFn: defaultConfigCluster, + BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + AWSSession: e2eCtx.BootstrapUserAWSSession, + Namespace: namespace, + ClusterName: clusterName, + Cleanup: true, + BYOLaunchTemplate: byoLT, + } + }) + ginkgo.By(fmt.Sprintf("getting cluster with name %s", clusterName)) cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{ Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), diff --git a/test/e2e/suites/managed/helpers.go b/test/e2e/suites/managed/helpers.go index cb6350ffb4..486f426634 100644 --- a/test/e2e/suites/managed/helpers.go +++ b/test/e2e/suites/managed/helpers.go @@ -27,6 +27,8 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/autoscaling" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/eks" ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types" "github.com/aws/aws-sdk-go-v2/service/iam" @@ -42,21 +44,22 @@ import ( // EKS related constants. const ( - EKSManagedPoolFlavor = "eks-managedmachinepool" - EKSControlPlaneOnlyFlavor = "eks-control-plane-only" - EKSControlPlaneOnlyWithAddonFlavor = "eks-control-plane-only-withaddon" - EKSMachineDeployOnlyFlavor = "eks-machine-deployment-only" - EKSManagedMachinePoolOnlyFlavor = "eks-managed-machinepool-only" - EKSManagedMachinePoolWithLaunchTemplateOnlyFlavor = "eks-managed-machinepool-with-launch-template-only" - EKSMachinePoolOnlyFlavor = "eks-machinepool-only" - EKSIPv6ClusterFlavor = "eks-ipv6-cluster" - EKSUpgradePolicyFlavor = "eks-upgrade-policy" - EKSControlPlaneOnlyLegacyFlavor = "eks-control-plane-only-legacy" - EKSClusterClassFlavor = "eks-clusterclass" - EKSAuthAPIAndConfigMapFlavor = "eks-auth-api-and-config-map" - EKSAuthBootstrapDisabledFlavor = "eks-auth-bootstrap-disabled" - EKSControlPlaneOnlyWithAccessEntriesFlavor = "eks-control-plane-only-with-accessentries" - EKSNodeadmClusterClassFlavor = "eks-nodeadm-clusterclass" + EKSManagedPoolFlavor = "eks-managedmachinepool" + EKSControlPlaneOnlyFlavor = "eks-control-plane-only" + EKSControlPlaneOnlyWithAddonFlavor = "eks-control-plane-only-withaddon" + EKSMachineDeployOnlyFlavor = "eks-machine-deployment-only" + EKSManagedMachinePoolOnlyFlavor = "eks-managed-machinepool-only" + EKSManagedMachinePoolWithLaunchTemplateOnlyFlavor = "eks-managed-machinepool-with-launch-template-only" + EKSManagedMachinePoolWithBYOLaunchTemplateOnlyFlavor = "eks-managed-machinepool-with-byo-launch-template-only" + EKSMachinePoolOnlyFlavor = "eks-machinepool-only" + EKSIPv6ClusterFlavor = "eks-ipv6-cluster" + EKSUpgradePolicyFlavor = "eks-upgrade-policy" + EKSControlPlaneOnlyLegacyFlavor = "eks-control-plane-only-legacy" + EKSClusterClassFlavor = "eks-clusterclass" + EKSAuthAPIAndConfigMapFlavor = "eks-auth-api-and-config-map" + EKSAuthBootstrapDisabledFlavor = "eks-auth-bootstrap-disabled" + EKSControlPlaneOnlyWithAccessEntriesFlavor = "eks-control-plane-only-with-accessentries" + EKSNodeadmClusterClassFlavor = "eks-nodeadm-clusterclass" ) const ( @@ -78,6 +81,10 @@ func getEKSNodegroupWithLaunchTemplateName(namespace, clusterName string) string return fmt.Sprintf("%s_%s-pool-lt-0", namespace, clusterName) } +func getEKSNodegroupWithBYOLaunchTemplateName(namespace, clusterName string) string { + return fmt.Sprintf("%s_%s-pool-byo-lt-0", namespace, clusterName) +} + func getControlPlaneName(clusterName string) string { return fmt.Sprintf("%s-control-plane", clusterName) } @@ -332,3 +339,65 @@ func verifyAccessEntries(ctx context.Context, eksClusterName string, expectedEnt } } } + +// BYOLaunchTemplate holds the ID and version of a launch template created for BYO tests. +type BYOLaunchTemplate struct { + ID string + Version int64 +} + +// CreateBYOLaunchTemplate creates a minimal EC2 launch template for use in BYO nodegroup tests. +// The template has no instance type or AMI so that EKS can resolve the defaults for the nodegroup. +// Callers must call DeleteBYOLaunchTemplate when done. +func CreateBYOLaunchTemplate(ctx context.Context, name string, sess *aws.Config) BYOLaunchTemplate { + ec2Client := ec2.NewFromConfig(*sess) + out, err := ec2Client.CreateLaunchTemplate(ctx, &ec2.CreateLaunchTemplateInput{ + LaunchTemplateName: aws.String(name), + LaunchTemplateData: &ec2types.RequestLaunchTemplateData{ + // Minimal data — EKS will supply instance type and AMI from the nodegroup spec. + MetadataOptions: &ec2types.LaunchTemplateInstanceMetadataOptionsRequest{ + HttpTokens: ec2types.LaunchTemplateHttpTokensStateRequired, + }, + }, + }) + Expect(err).NotTo(HaveOccurred(), "failed to create BYO launch template") + Expect(out.LaunchTemplate).NotTo(BeNil()) + return BYOLaunchTemplate{ + ID: aws.ToString(out.LaunchTemplate.LaunchTemplateId), + Version: aws.ToInt64(out.LaunchTemplate.LatestVersionNumber), + } +} + +// DeleteBYOLaunchTemplate deletes the EC2 launch template created by CreateBYOLaunchTemplate. +func DeleteBYOLaunchTemplate(ctx context.Context, id string, sess *aws.Config) { + ec2Client := ec2.NewFromConfig(*sess) + _, err := ec2Client.DeleteLaunchTemplate(ctx, &ec2.DeleteLaunchTemplateInput{ + LaunchTemplateId: aws.String(id), + }) + Expect(err).NotTo(HaveOccurred(), "failed to delete BYO launch template %s", id) +} + +// verifyManagedNodeGroupUsesBYOLaunchTemplate asserts that the given nodegroup is ACTIVE and that +// its launch template ID matches the expected BYO launch template ID. +func verifyManagedNodeGroupUsesBYOLaunchTemplate(ctx context.Context, eksClusterName, nodeGroupName, expectedLaunchTemplateID string, sess *aws.Config) { + eksClient := eks.NewFromConfig(*sess) + var result *eks.DescribeNodegroupOutput + Eventually(func() error { + var err error + result, err = eksClient.DescribeNodegroup(ctx, &eks.DescribeNodegroupInput{ + ClusterName: aws.String(eksClusterName), + NodegroupName: aws.String(nodeGroupName), + }) + if err != nil { + return fmt.Errorf("error describing nodegroup: %w", err) + } + if result.Nodegroup.Status != ekstypes.NodegroupStatusActive { + return fmt.Errorf("expected nodegroup status %q, got %q", ekstypes.NodegroupStatusActive, result.Nodegroup.Status) + } + return nil + }, clientRequestTimeout, clientRequestCheckInterval).Should(Succeed(), "nodegroup did not become ACTIVE") + + Expect(result.Nodegroup.LaunchTemplate).NotTo(BeNil(), "expected nodegroup to have a launch template") + Expect(aws.ToString(result.Nodegroup.LaunchTemplate.Id)).To(Equal(expectedLaunchTemplateID), + "nodegroup launch template ID does not match the BYO template ID") +} diff --git a/test/e2e/suites/managed/machine_pool.go b/test/e2e/suites/managed/machine_pool.go index 429f82a5b7..79467adf15 100644 --- a/test/e2e/suites/managed/machine_pool.go +++ b/test/e2e/suites/managed/machine_pool.go @@ -23,6 +23,7 @@ import ( "context" "encoding/base64" "fmt" + "strconv" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -145,3 +146,79 @@ func MachinePoolSpec(ctx context.Context, inputGetter func() MachinePoolSpecInpu }, input.E2EConfig.GetIntervals("", "wait-delete-machine-pool")...) } } + +// BYOMachinePoolSpecInput is the input for BYOMachinePoolSpec. +type BYOMachinePoolSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ConfigClusterFn DefaultConfigClusterFn + BootstrapClusterProxy framework.ClusterProxy + AWSSession *aws.Config + Namespace *corev1.Namespace + ClusterName string + Cleanup bool + // BYOLaunchTemplate is a pre-created launch template to reference. + BYOLaunchTemplate BYOLaunchTemplate +} + +// BYOMachinePoolSpec tests creating an EKS managed node group that references a user-provided +// (BYO) launch template. CAPA must not create, update, or delete the launch template. +func BYOMachinePoolSpec(ctx context.Context, inputGetter func() BYOMachinePoolSpecInput) { + input := inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil") + Expect(input.ConfigClusterFn).ToNot(BeNil(), "Invalid argument. input.ConfigClusterFn can't be nil") + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil") + Expect(input.AWSSession).ToNot(BeNil(), "Invalid argument. input.AWSSession can't be nil") + Expect(input.Namespace).NotTo(BeNil(), "Invalid argument. input.Namespace can't be nil") + Expect(input.ClusterName).ShouldNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty") + Expect(input.BYOLaunchTemplate.ID).ShouldNot(BeEmpty(), "Invalid argument. BYOLaunchTemplate.ID can't be empty") + + ginkgo.By(fmt.Sprintf("getting cluster with name %s", input.ClusterName)) + cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{ + Getter: input.BootstrapClusterProxy.GetClient(), + Namespace: input.Namespace.Name, + Name: input.ClusterName, + }) + Expect(cluster).NotTo(BeNil(), "couldn't find CAPI cluster") + + ginkgo.By(fmt.Sprintf("applying the %s template with BYO launch template %s@v%d", + EKSManagedMachinePoolWithBYOLaunchTemplateOnlyFlavor, + input.BYOLaunchTemplate.ID, input.BYOLaunchTemplate.Version)) + configCluster := input.ConfigClusterFn(input.ClusterName, input.Namespace.Name) + configCluster.Flavor = EKSManagedMachinePoolWithBYOLaunchTemplateOnlyFlavor + configCluster.WorkerMachineCount = ptr.To[int64](1) + workloadClusterTemplate := shared.GetTemplate(ctx, configCluster) + + // Substitute the BYO launch template placeholders in the cluster template. + workloadClusterTemplate = []byte(strings.ReplaceAll(string(workloadClusterTemplate), + "${BYO_LAUNCH_TEMPLATE_ID}", input.BYOLaunchTemplate.ID)) + workloadClusterTemplate = []byte(strings.ReplaceAll(string(workloadClusterTemplate), + "${BYO_LAUNCH_TEMPLATE_VERSION}", strconv.FormatInt(input.BYOLaunchTemplate.Version, 10))) + + ginkgo.By(fmt.Sprintf("Applying the %s cluster template yaml to the cluster", configCluster.Flavor)) + err := input.BootstrapClusterProxy.CreateOrUpdate(ctx, workloadClusterTemplate) + Expect(err).ShouldNot(HaveOccurred()) + + ginkgo.By("Waiting for the machine pool to be running") + mp := framework.DiscoveryAndWaitForMachinePools(ctx, framework.DiscoveryAndWaitForMachinePoolsInput{ + Lister: input.BootstrapClusterProxy.GetClient(), + Getter: input.BootstrapClusterProxy.GetClient(), + Cluster: cluster, + }, input.E2EConfig.GetIntervals("", "wait-worker-nodes")...) + Expect(len(mp)).To(Equal(1)) + + ginkgo.By("Verifying the node group uses the BYO launch template") + eksClusterName := getEKSClusterName(input.Namespace.Name, input.ClusterName) + nodeGroupName := getEKSNodegroupWithBYOLaunchTemplateName(input.Namespace.Name, input.ClusterName) + verifyManagedNodeGroupUsesBYOLaunchTemplate(ctx, eksClusterName, nodeGroupName, input.BYOLaunchTemplate.ID, input.AWSSession) + + if input.Cleanup { + deleteMachinePool(ctx, deleteMachinePoolInput{ + Deleter: input.BootstrapClusterProxy.GetClient(), + MachinePool: mp[0], + }) + waitForMachinePoolDeleted(ctx, waitForMachinePoolDeletedInput{ + Getter: input.BootstrapClusterProxy.GetClient(), + MachinePool: mp[0], + }, input.E2EConfig.GetIntervals("", "wait-delete-machine-pool")...) + } +}