From decde29d621f9619aafcb09410244d24c426324b Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Thu, 2 Jul 2026 16:49:58 -0700 Subject: [PATCH 01/25] feat: add scriptless containerd 2.0 support --- aks-node-controller/parser/helper.go | 41 +++- aks-node-controller/parser/helper_test.go | 185 ++++++++++++++++++ .../parser/templates/containerd_v2.toml.gtpl | 73 +++++++ .../templates/containerd_v2_no_GPU.toml.gtpl | 52 +++++ e2e/node_config.go | 12 ++ e2e/scenario_test.go | 3 - pkg/agent/baker.go | 122 ++++++------ 7 files changed, 419 insertions(+), 69 deletions(-) create mode 100644 aks-node-controller/parser/templates/containerd_v2.toml.gtpl create mode 100644 aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 168cfc48a80..41fc83c7cd9 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -51,6 +51,18 @@ var ( containerdConfigNoGPUTemplate = template.Must( template.New("nogpucontainerdconfig").Funcs(getFuncMapForContainerdConfigTemplate()).Parse(containerdConfigNoGPUTemplateText), ) + //go:embed templates/containerd_v2.toml.gtpl + containerdV2ConfigTemplateText string + //nolint:gochecknoglobals + containerdV2ConfigTemplate = template.Must( + template.New("containerdv2config").Funcs(getFuncMapForContainerdConfigTemplate()).Parse(containerdV2ConfigTemplateText), + ) + //go:embed templates/containerd_v2_no_GPU.toml.gtpl + containerdV2ConfigNoGPUTemplateText string + //nolint:gochecknoglobals + containerdV2ConfigNoGPUTemplate = template.Must( + template.New("nogpucontainerdv2config").Funcs(getFuncMapForContainerdConfigTemplate()).Parse(containerdV2ConfigNoGPUTemplateText), + ) //go:embed templates/localdns.toml.gtpl localDnsCorefileTemplateText string @@ -181,11 +193,20 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat return "", fmt.Errorf("AKSNodeConfig is nil") } - // TODO: add containerdv2 support - // the containerd config template is different based on whether the node is with GPU or not. - _template := containerdConfigTemplate - if noGPU { - _template = containerdConfigNoGPUTemplate + // Select the appropriate containerd config template based on version and GPU presence. + // Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images/runtime) + // compared to containerd 1.x (io.containerd.grpc.v1.cri). + var _template *template.Template + if isContainerdV2(aksnodeconfig.GetContainerdConfig().GetContainerdVersion()) { + _template = containerdV2ConfigTemplate + if noGPU { + _template = containerdV2ConfigNoGPUTemplate + } + } else { + _template = containerdConfigTemplate + if noGPU { + _template = containerdConfigNoGPUTemplate + } } var buffer bytes.Buffer @@ -196,6 +217,16 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat return buffer.String(), nil } +// isContainerdV2 returns true if the containerd version string indicates a 2.x release. +// Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images and +// io.containerd.cri.v1.runtime) compared to 1.x (io.containerd.grpc.v1.cri). +func isContainerdV2(version string) bool { + if version == "" { + return false + } + return helpers.IsKubernetesVersionGe(version, "2.0.0") +} + func getIsMIGNode(gpuInstanceProfile string) bool { return gpuInstanceProfile != "" } diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index e4b17a46846..75dec79aad2 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -506,6 +506,191 @@ oom_score = -999 } } +func Test_isContainerdV2(t *testing.T) { + tests := []struct { + name string + version string + want bool + }{ + {name: "empty version", version: "", want: false}, + {name: "v1.6.x", version: "1.6.28", want: false}, + {name: "v1.7.x", version: "1.7.22", want: false}, + {name: "v2.0.0", version: "2.0.0", want: true}, + {name: "v2.0.1", version: "2.0.1", want: true}, + {name: "v2.1.0", version: "2.1.0", want: true}, + {name: "invalid version", version: "notaversion", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isContainerdV2(tt.version); got != tt.want { + t.Errorf("isContainerdV2(%q) = %v, want %v", tt.version, got, tt.want) + } + }) + } +} + +func Test_getContainerdConfigV2(t *testing.T) { + type args struct { + aksnodeconfig *aksnodeconfigv1.Configuration + noGpu bool + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Containerd v2 default config", + args: args{ + aksnodeconfig: &aksnodeconfigv1.Configuration{ + NeedsCgroupv2: to.Ptr(true), + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ + ContainerdVersion: "2.0.0", + }, + }, + }, + want: base64.StdEncoding.EncodeToString([]byte(`version = 2 +oom_score = -999 +[plugins."io.containerd.cri.v1.images"] +[plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "" +[plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] +[plugins."io.containerd.cri.v1.runtime".containerd] + default_runtime_name = "runc" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" +[metrics] + address = "0.0.0.0:10257" +`)), + }, + { + name: "Containerd v2 with GPU", + args: args{ + aksnodeconfig: &aksnodeconfigv1.Configuration{ + NeedsCgroupv2: to.Ptr(true), + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ + ContainerdVersion: "2.0.1", + }, + GpuConfig: &aksnodeconfigv1.GpuConfig{ + EnableNvidia: to.Ptr(true), + }, + }, + noGpu: false, + }, + want: base64.StdEncoding.EncodeToString([]byte(`version = 2 +oom_score = -999 +[plugins."io.containerd.cri.v1.images"] +[plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "" +[plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] +[plugins."io.containerd.cri.v1.runtime".containerd] + default_runtime_name = "nvidia-container-runtime" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/nvidia-container-runtime" +[metrics] + address = "0.0.0.0:10257" +`)), + }, + { + name: "Containerd v2 no GPU template", + args: args{ + aksnodeconfig: &aksnodeconfigv1.Configuration{ + NeedsCgroupv2: to.Ptr(true), + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ + ContainerdVersion: "2.0.0", + }, + GpuConfig: &aksnodeconfigv1.GpuConfig{ + EnableNvidia: to.Ptr(true), + }, + }, + noGpu: true, + }, + want: base64.StdEncoding.EncodeToString([]byte(`version = 2 +oom_score = -999 +[plugins."io.containerd.cri.v1.images"] +[plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "" +[plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] +[plugins."io.containerd.cri.v1.runtime".containerd] + default_runtime_name = "runc" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" +[metrics] + address = "0.0.0.0:10257" +`)), + }, + { + name: "Containerd v1 still uses old templates", + args: args{ + aksnodeconfig: &aksnodeconfigv1.Configuration{ + NeedsCgroupv2: to.Ptr(true), + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ + ContainerdVersion: "1.7.22", + }, + }, + }, + want: base64.StdEncoding.EncodeToString([]byte(`version = 2 +oom_score = -999 +[plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "" + enable_cdi = true + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" + [plugins."io.containerd.grpc.v1.cri".registry.headers] + X-Meta-Source-Client = ["azure/aks"] +[metrics] + address = "0.0.0.0:10257" +`)), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got string + if tt.args.noGpu { + got = getNoGPUContainerdConfigBase64(tt.args.aksnodeconfig) + } else { + got = getContainerdConfigBase64(tt.args.aksnodeconfig) + } + if got != tt.want { + t.Errorf("getContainerdConfig() = %v, want %v", got, tt.want) + } + }) + } +} + func Test_getKubenetTemplate(t *testing.T) { tests := []struct { name string diff --git a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl new file mode 100644 index 00000000000..6515519bd6e --- /dev/null +++ b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl @@ -0,0 +1,73 @@ +version = 2 +oom_score = -999{{if getHasDataDir .KubeletConfig}} +root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} +[plugins."io.containerd.cri.v1.images"] +{{- if .GetEnableArtifactStreaming }} + snapshotter = "overlaybd" + disable_snapshot_annotations = false +{{- end}} + [plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" + {{- if isKubernetesVersionGe .GetKubernetesVersion "1.22.0"}} + [plugins."io.containerd.cri.v1.images".registry] + config_path = "/etc/containerd/certs.d" + {{- end}} + [plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] +[plugins."io.containerd.cri.v1.runtime".containerd] + {{- if getEnableNvidia . }} + default_runtime_name = "nvidia-container-runtime" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + {{- else}} + default_runtime_name = "runc" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" + {{- end}} + {{- if getEnsureNoDupePromiscuousBridge .GetNetworkConfig }} + [plugins."io.containerd.cri.v1.runtime".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "/etc/containerd/kubenet_template.conf" + {{- end}} +[metrics] + address = "0.0.0.0:10257" +{{- if .GetEnableArtifactStreaming }} +[proxy_plugins] + [proxy_plugins.overlaybd] + type = "snapshot" + address = "/run/overlaybd-snapshotter/overlaybd.sock" +{{- end}} +{{- if .GetIsKata }} +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" +[proxy_plugins] + [proxy_plugins.tardev] + type = "snapshot" + address = "/run/containerd/tardev-snapshotter.sock" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc] + snapshotter = "tardev" + runtime_type = "io.containerd.kata-cc.v2" + privileged_without_host_devices = true + pod_annotations = ["io.katacontainers.*"] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options] + ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" +{{- end}} + diff --git a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl new file mode 100644 index 00000000000..45a2129f743 --- /dev/null +++ b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl @@ -0,0 +1,52 @@ +version = 2 +oom_score = -999{{if getHasDataDir .KubeletConfig}} +root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} +[plugins."io.containerd.cri.v1.images"] +{{- if .GetEnableArtifactStreaming }} + snapshotter = "overlaybd" + disable_snapshot_annotations = false +{{- end}} + [plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" + {{- if isKubernetesVersionGe .GetKubernetesVersion "1.22.0"}} + [plugins."io.containerd.cri.v1.images".registry] + config_path = "/etc/containerd/certs.d" + {{- end}} + [plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] +[plugins."io.containerd.cri.v1.runtime".containerd] + default_runtime_name = "runc" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" + {{- if getEnsureNoDupePromiscuousBridge .GetNetworkConfig }} + [plugins."io.containerd.cri.v1.runtime".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "/etc/containerd/kubenet_template.conf" + {{- end}} +[metrics] + address = "0.0.0.0:10257" +{{- if .GetEnableArtifactStreaming }} +[proxy_plugins] + [proxy_plugins.overlaybd] + type = "snapshot" + address = "/run/overlaybd-snapshotter/overlaybd.sock" +{{- end}} +{{- if .GetIsKata }} +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] + ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml" +[proxy_plugins] + [proxy_plugins.tardev] + type = "snapshot" + address = "/run/containerd/tardev-snapshotter.sock" +{{- end}} diff --git a/e2e/node_config.go b/e2e/node_config.go index 3472d383501..87333eda500 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -265,6 +265,7 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod KubernetesVersion: cs.Properties.OrchestratorProfile.OrchestratorVersion, ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdDownloadUrlBase: nbc.CloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, + ContainerdVersion: getContainerdVersionFromDistro(nbc.AgentPoolProfile), }, OutboundCommand: helpers.GetDefaultOutboundCommand(), KubernetesCaCert: base64.StdEncoding.EncodeToString([]byte(cs.Properties.CertificateProfile.CaCertificate)), @@ -1087,3 +1088,14 @@ func pruneKubeletConfig(kubernetesVersion string, datamodel *datamodel.NodeBoots } return datamodel, nil } + +// getContainerdVersionFromDistro returns a containerd version string based on the distro, +// matching baker.go's logic where Is2404VHDDistro() selects the v2 containerd config template. +// This allows the aks-node-controller to select the correct containerd config template (v1 vs v2) +// based on the containerd version string. +func getContainerdVersionFromDistro(profile *datamodel.AgentPoolProfile) string { + if profile != nil && profile.Is2404VHDDistro() { + return "2.0.0" + } + return "" +} diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 40d93503dd3..69101aa5025 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -686,9 +686,6 @@ func Test_Ubuntu2204_Early_Failure_Scriptless(t *testing.T) { func Test_Ubuntu2404_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ Description: "testing that a new ubuntu 2404 node using self contained installer can be properly bootstrapped", - Tags: Tags{ - Scriptless: true, - }, Config: Config{ Cluster: ClusterKubenet, VHD: config.VHDUbuntu2404Gen2Containerd, diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 2e8b4d6e101..16760679327 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -1807,44 +1807,44 @@ root = "{{GetDataDir}}"{{- end}} snapshotter = "overlaybd" disable_snapshot_annotations = false {{- end}} -[plugins."io.containerd.cri.v1.images".pinned_images] - sandbox = "{{GetPodInfraContainerSpec}}" -{{- if IsKubernetesVersionGe "1.22.0"}} -[plugins."io.containerd.cri.v1.images".registry] - config_path = "/etc/containerd/certs.d" -{{- end}} -[plugins."io.containerd.cri.v1.images".registry.headers] - X-Meta-Source-Client = ["azure/aks"] + [plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "{{GetPodInfraContainerSpec}}" + {{- if IsKubernetesVersionGe "1.22.0"}} + [plugins."io.containerd.cri.v1.images".registry] + config_path = "/etc/containerd/certs.d" + {{- end}} + [plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] [plugins."io.containerd.cri.v1.runtime".containerd] - {{- if IsNSeriesSKU }} - default_runtime_name = "nvidia-container-runtime" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options] - BinaryName = "/usr/bin/nvidia-container-runtime" - SystemdCgroup = true - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] - BinaryName = "/usr/bin/nvidia-container-runtime" -{{- else}} - default_runtime_name = "runc" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] - BinaryName = "/usr/bin/runc" - SystemdCgroup = true - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] - BinaryName = "/usr/bin/runc" -{{- end}} -{{- if and (IsKubenet) (not HasCalicoNetworkPolicy) }} -[plugins."io.containerd.cri.v1.runtime".cni] - bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - conf_template = "/etc/containerd/kubenet_template.conf" -{{- end}} + {{- if IsNSeriesSKU }} + default_runtime_name = "nvidia-container-runtime" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + {{- else}} + default_runtime_name = "runc" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" + {{- end}} + {{- if and (IsKubenet) (not HasCalicoNetworkPolicy) }} + [plugins."io.containerd.cri.v1.runtime".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "/etc/containerd/kubenet_template.conf" + {{- end}} [metrics] address = "0.0.0.0:10257" {{- if IsArtifactStreamingEnabled }} @@ -1880,31 +1880,31 @@ root = "{{GetDataDir}}"{{- end}} snapshotter = "overlaybd" disable_snapshot_annotations = false {{- end}} -[plugins."io.containerd.cri.v1.images".pinned_images] - sandbox = "{{GetPodInfraContainerSpec}}" -{{- if IsKubernetesVersionGe "1.22.0"}} -[plugins."io.containerd.cri.v1.images".registry] - config_path = "/etc/containerd/certs.d" -{{- end}} -[plugins."io.containerd.cri.v1.images".registry.headers] - X-Meta-Source-Client = ["azure/aks"] + [plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "{{GetPodInfraContainerSpec}}" + {{- if IsKubernetesVersionGe "1.22.0"}} + [plugins."io.containerd.cri.v1.images".registry] + config_path = "/etc/containerd/certs.d" + {{- end}} + [plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] [plugins."io.containerd.cri.v1.runtime".containerd] - default_runtime_name = "runc" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] - BinaryName = "/usr/bin/runc" - SystemdCgroup = true - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] - BinaryName = "/usr/bin/runc" -{{- if and (IsKubenet) (not HasCalicoNetworkPolicy) }} -[plugins."io.containerd.cri.v1.runtime".cni] - bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - conf_template = "/etc/containerd/kubenet_template.conf" -{{- end}} + default_runtime_name = "runc" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" + {{- if and (IsKubenet) (not HasCalicoNetworkPolicy) }} + [plugins."io.containerd.cri.v1.runtime".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "/etc/containerd/kubenet_template.conf" + {{- end}} [metrics] address = "0.0.0.0:10257" {{- if IsArtifactStreamingEnabled }} From 1cd9637067f928cc0441556ace2a3ab05f6417c4 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Wed, 8 Jul 2026 00:09:53 -0700 Subject: [PATCH 02/25] add usecontainerdv2 to aksnodeconfig --- .../aksnodeconfig/v1/containerd_config.pb.go | 28 +++++++++++++------ .../aksnodeconfig/v1/containerd_config.proto | 3 ++ e2e/node_config.go | 13 +-------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go b/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go index 54d3a35de59..642d3df14e6 100644 --- a/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go +++ b/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go @@ -31,6 +31,8 @@ type ContainerdConfig struct { ContainerdVersion string `protobuf:"bytes,2,opt,name=containerd_version,json=containerdVersion,proto3" json:"containerd_version,omitempty"` // The URL for downloading the containerd package. ContainerdPackageUrl string `protobuf:"bytes,3,opt,name=containerd_package_url,json=containerdPackageUrl,proto3" json:"containerd_package_url,omitempty"` + // Specify if use containerd 2.0, default to false. + UseContainerdV2 bool `protobuf:"varint,4,opt,name=use_containerd_v2,json=useContainerdV2,proto3" json:"use_containerd_v2,omitempty"` } func (x *ContainerdConfig) Reset() { @@ -84,13 +86,20 @@ func (x *ContainerdConfig) GetContainerdPackageUrl() string { return "" } +func (x *ContainerdConfig) GetUseContainerdV2() bool { + if x != nil { + return x.UseContainerdV2 + } + return false +} + var File_aksnodeconfig_v1_containerd_config_proto protoreflect.FileDescriptor var file_aksnodeconfig_v1_containerd_config_proto_rawDesc = []byte{ 0x0a, 0x28, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x10, 0x61, 0x6b, 0x73, 0x6e, - 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x22, 0xb8, 0x01, 0x0a, + 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x22, 0xe4, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3f, 0x0a, 0x1c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x5f, 0x62, 0x61, 0x73, @@ -102,13 +111,16 @@ var file_aksnodeconfig_v1_containerd_config_proto_rawDesc = []byte{ 0x6e, 0x12, 0x34, 0x0a, 0x16, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x50, 0x61, 0x63, - 0x6b, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, - 0x74, 0x62, 0x61, 0x6b, 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, - 0x65, 0x6e, 0x2f, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6b, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x2a, 0x0a, 0x11, 0x75, 0x73, 0x65, 0x5f, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x76, 0x32, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0f, 0x75, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x64, 0x56, 0x32, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, + 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, + 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, + 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto b/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto index 7a528c8f726..8968e654819 100644 --- a/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto +++ b/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto @@ -12,4 +12,7 @@ message ContainerdConfig { // The URL for downloading the containerd package. string containerd_package_url = 3; + + // Specify if use containerd 2.0, default to false. + bool use_containerd_v2 = 4; } diff --git a/e2e/node_config.go b/e2e/node_config.go index 87333eda500..89f36c2e084 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -265,7 +265,7 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod KubernetesVersion: cs.Properties.OrchestratorProfile.OrchestratorVersion, ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdDownloadUrlBase: nbc.CloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, - ContainerdVersion: getContainerdVersionFromDistro(nbc.AgentPoolProfile), + UseContainerdV2: nbc.AgentPoolProfile.Is2404VHDDistro(), }, OutboundCommand: helpers.GetDefaultOutboundCommand(), KubernetesCaCert: base64.StdEncoding.EncodeToString([]byte(cs.Properties.CertificateProfile.CaCertificate)), @@ -1088,14 +1088,3 @@ func pruneKubeletConfig(kubernetesVersion string, datamodel *datamodel.NodeBoots } return datamodel, nil } - -// getContainerdVersionFromDistro returns a containerd version string based on the distro, -// matching baker.go's logic where Is2404VHDDistro() selects the v2 containerd config template. -// This allows the aks-node-controller to select the correct containerd config template (v1 vs v2) -// based on the containerd version string. -func getContainerdVersionFromDistro(profile *datamodel.AgentPoolProfile) string { - if profile != nil && profile.Is2404VHDDistro() { - return "2.0.0" - } - return "" -} From 93a0b79fffcf6c2067f237f087cf740afbeec6da Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:01:00 +0000 Subject: [PATCH 03/25] Fix Test_getContainerdConfigV2 expected strings to match actual template indentation Co-authored-by: lilypan26 <106703606+lilypan26@users.noreply.github.com> --- aks-node-controller/parser/helper_test.go | 86 ++++++++++++----------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 75dec79aad2..8cf100847da 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -552,23 +552,24 @@ func Test_getContainerdConfigV2(t *testing.T) { want: base64.StdEncoding.EncodeToString([]byte(`version = 2 oom_score = -999 [plugins."io.containerd.cri.v1.images"] -[plugins."io.containerd.cri.v1.images".pinned_images] - sandbox = "" -[plugins."io.containerd.cri.v1.images".registry.headers] - X-Meta-Source-Client = ["azure/aks"] + [plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "" + [plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] [plugins."io.containerd.cri.v1.runtime".containerd] - default_runtime_name = "runc" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] - BinaryName = "/usr/bin/runc" - SystemdCgroup = true - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] - BinaryName = "/usr/bin/runc" + default_runtime_name = "runc" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" [metrics] address = "0.0.0.0:10257" + `)), }, { @@ -588,23 +589,24 @@ oom_score = -999 want: base64.StdEncoding.EncodeToString([]byte(`version = 2 oom_score = -999 [plugins."io.containerd.cri.v1.images"] -[plugins."io.containerd.cri.v1.images".pinned_images] - sandbox = "" -[plugins."io.containerd.cri.v1.images".registry.headers] - X-Meta-Source-Client = ["azure/aks"] + [plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "" + [plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] [plugins."io.containerd.cri.v1.runtime".containerd] - default_runtime_name = "nvidia-container-runtime" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options] - BinaryName = "/usr/bin/nvidia-container-runtime" - SystemdCgroup = true - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] - BinaryName = "/usr/bin/nvidia-container-runtime" + default_runtime_name = "nvidia-container-runtime" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/nvidia-container-runtime" [metrics] address = "0.0.0.0:10257" + `)), }, { @@ -624,21 +626,21 @@ oom_score = -999 want: base64.StdEncoding.EncodeToString([]byte(`version = 2 oom_score = -999 [plugins."io.containerd.cri.v1.images"] -[plugins."io.containerd.cri.v1.images".pinned_images] - sandbox = "" -[plugins."io.containerd.cri.v1.images".registry.headers] - X-Meta-Source-Client = ["azure/aks"] + [plugins."io.containerd.cri.v1.images".pinned_images] + sandbox = "" + [plugins."io.containerd.cri.v1.images".registry.headers] + X-Meta-Source-Client = ["azure/aks"] [plugins."io.containerd.cri.v1.runtime".containerd] - default_runtime_name = "runc" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] - BinaryName = "/usr/bin/runc" - SystemdCgroup = true - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] - BinaryName = "/usr/bin/runc" + default_runtime_name = "runc" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + SystemdCgroup = true + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" [metrics] address = "0.0.0.0:10257" `)), From bd65dc6e15ec9067527315dcc352cf21828348a9 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Wed, 8 Jul 2026 11:17:15 -0700 Subject: [PATCH 04/25] fix getcontainerconfigtemplate --- aks-node-controller/parser/helper.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 41fc83c7cd9..655d7ad39bc 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -197,7 +197,7 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat // Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images/runtime) // compared to containerd 1.x (io.containerd.grpc.v1.cri). var _template *template.Template - if isContainerdV2(aksnodeconfig.GetContainerdConfig().GetContainerdVersion()) { + if aksnodeconfig.GetContainerdConfig().GetUseContainerdV2() { _template = containerdV2ConfigTemplate if noGPU { _template = containerdV2ConfigNoGPUTemplate @@ -217,16 +217,6 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat return buffer.String(), nil } -// isContainerdV2 returns true if the containerd version string indicates a 2.x release. -// Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images and -// io.containerd.cri.v1.runtime) compared to 1.x (io.containerd.grpc.v1.cri). -func isContainerdV2(version string) bool { - if version == "" { - return false - } - return helpers.IsKubernetesVersionGe(version, "2.0.0") -} - func getIsMIGNode(gpuInstanceProfile string) bool { return gpuInstanceProfile != "" } From fb0f507bf838e0228a383a123db90b43b1fafcc7 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Wed, 8 Jul 2026 15:14:37 -0700 Subject: [PATCH 05/25] fix UTs --- aks-node-controller/parser/helper_test.go | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 8cf100847da..6660bc6bf31 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -506,29 +506,6 @@ oom_score = -999 } } -func Test_isContainerdV2(t *testing.T) { - tests := []struct { - name string - version string - want bool - }{ - {name: "empty version", version: "", want: false}, - {name: "v1.6.x", version: "1.6.28", want: false}, - {name: "v1.7.x", version: "1.7.22", want: false}, - {name: "v2.0.0", version: "2.0.0", want: true}, - {name: "v2.0.1", version: "2.0.1", want: true}, - {name: "v2.1.0", version: "2.1.0", want: true}, - {name: "invalid version", version: "notaversion", want: false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isContainerdV2(tt.version); got != tt.want { - t.Errorf("isContainerdV2(%q) = %v, want %v", tt.version, got, tt.want) - } - }) - } -} - func Test_getContainerdConfigV2(t *testing.T) { type args struct { aksnodeconfig *aksnodeconfigv1.Configuration From d63bfeeeae96db9b5676aff4c1fdb91d32d7c4c3 Mon Sep 17 00:00:00 2001 From: lilypan26 Date: Thu, 9 Jul 2026 09:51:34 -0700 Subject: [PATCH 06/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- aks-node-controller/parser/helper_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 6660bc6bf31..9e1cf392a68 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -589,10 +589,9 @@ oom_score = -999 { name: "Containerd v2 no GPU template", args: args{ - aksnodeconfig: &aksnodeconfigv1.Configuration{ - NeedsCgroupv2: to.Ptr(true), ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdVersion: "2.0.0", + UseContainerdV2: true, }, GpuConfig: &aksnodeconfigv1.GpuConfig{ EnableNvidia: to.Ptr(true), From 96de66e20cf385b1417e44b81337aa6eb7d3c40c Mon Sep 17 00:00:00 2001 From: lilypan26 Date: Thu, 9 Jul 2026 09:52:18 -0700 Subject: [PATCH 07/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- aks-node-controller/parser/helper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 9e1cf392a68..63b24a8ed1e 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -520,9 +520,9 @@ func Test_getContainerdConfigV2(t *testing.T) { name: "Containerd v2 default config", args: args{ aksnodeconfig: &aksnodeconfigv1.Configuration{ - NeedsCgroupv2: to.Ptr(true), ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdVersion: "2.0.0", + UseContainerdV2: true, }, }, }, From f8b18a64f8082933de17a2c3836bb34a940cb17d Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Thu, 9 Jul 2026 11:10:04 -0700 Subject: [PATCH 08/25] fix UTs --- aks-node-controller/parser/helper_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 63b24a8ed1e..83e71c1c529 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -556,6 +556,7 @@ oom_score = -999 NeedsCgroupv2: to.Ptr(true), ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdVersion: "2.0.1", + UseContainerdV2: true, }, GpuConfig: &aksnodeconfigv1.GpuConfig{ EnableNvidia: to.Ptr(true), @@ -589,6 +590,7 @@ oom_score = -999 { name: "Containerd v2 no GPU template", args: args{ + aksnodeconfig: &aksnodeconfigv1.Configuration{ ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdVersion: "2.0.0", UseContainerdV2: true, From 69bbf8acf836436d1c67ecb11e1c475bbed3ed0f Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Thu, 9 Jul 2026 19:14:20 -0700 Subject: [PATCH 09/25] revert proto changes --- .../proto/aksnodeconfig/v1/containerd_config.proto | 3 --- e2e/node_config.go | 13 ++++++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto b/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto index 8968e654819..7a528c8f726 100644 --- a/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto +++ b/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto @@ -12,7 +12,4 @@ message ContainerdConfig { // The URL for downloading the containerd package. string containerd_package_url = 3; - - // Specify if use containerd 2.0, default to false. - bool use_containerd_v2 = 4; } diff --git a/e2e/node_config.go b/e2e/node_config.go index 89f36c2e084..87333eda500 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -265,7 +265,7 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod KubernetesVersion: cs.Properties.OrchestratorProfile.OrchestratorVersion, ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdDownloadUrlBase: nbc.CloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, - UseContainerdV2: nbc.AgentPoolProfile.Is2404VHDDistro(), + ContainerdVersion: getContainerdVersionFromDistro(nbc.AgentPoolProfile), }, OutboundCommand: helpers.GetDefaultOutboundCommand(), KubernetesCaCert: base64.StdEncoding.EncodeToString([]byte(cs.Properties.CertificateProfile.CaCertificate)), @@ -1088,3 +1088,14 @@ func pruneKubeletConfig(kubernetesVersion string, datamodel *datamodel.NodeBoots } return datamodel, nil } + +// getContainerdVersionFromDistro returns a containerd version string based on the distro, +// matching baker.go's logic where Is2404VHDDistro() selects the v2 containerd config template. +// This allows the aks-node-controller to select the correct containerd config template (v1 vs v2) +// based on the containerd version string. +func getContainerdVersionFromDistro(profile *datamodel.AgentPoolProfile) string { + if profile != nil && profile.Is2404VHDDistro() { + return "2.0.0" + } + return "" +} From a3fdd7b461a984d3120b06f09e9c471cef2979c6 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 9 Jul 2026 19:14:52 -0700 Subject: [PATCH 10/25] Revert "add usecontainerdv2 to aksnodeconfig" This reverts commit 1cd9637067f928cc0441556ace2a3ab05f6417c4. --- .../aksnodeconfig/v1/containerd_config.pb.go | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go b/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go index 642d3df14e6..54d3a35de59 100644 --- a/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go +++ b/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go @@ -31,8 +31,6 @@ type ContainerdConfig struct { ContainerdVersion string `protobuf:"bytes,2,opt,name=containerd_version,json=containerdVersion,proto3" json:"containerd_version,omitempty"` // The URL for downloading the containerd package. ContainerdPackageUrl string `protobuf:"bytes,3,opt,name=containerd_package_url,json=containerdPackageUrl,proto3" json:"containerd_package_url,omitempty"` - // Specify if use containerd 2.0, default to false. - UseContainerdV2 bool `protobuf:"varint,4,opt,name=use_containerd_v2,json=useContainerdV2,proto3" json:"use_containerd_v2,omitempty"` } func (x *ContainerdConfig) Reset() { @@ -86,20 +84,13 @@ func (x *ContainerdConfig) GetContainerdPackageUrl() string { return "" } -func (x *ContainerdConfig) GetUseContainerdV2() bool { - if x != nil { - return x.UseContainerdV2 - } - return false -} - var File_aksnodeconfig_v1_containerd_config_proto protoreflect.FileDescriptor var file_aksnodeconfig_v1_containerd_config_proto_rawDesc = []byte{ 0x0a, 0x28, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x10, 0x61, 0x6b, 0x73, 0x6e, - 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x22, 0xe4, 0x01, 0x0a, + 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x22, 0xb8, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3f, 0x0a, 0x1c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x5f, 0x62, 0x61, 0x73, @@ -111,16 +102,13 @@ var file_aksnodeconfig_v1_containerd_config_proto_rawDesc = []byte{ 0x6e, 0x12, 0x34, 0x0a, 0x16, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x50, 0x61, 0x63, - 0x6b, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x2a, 0x0a, 0x11, 0x75, 0x73, 0x65, 0x5f, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x76, 0x32, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0f, 0x75, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x64, 0x56, 0x32, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, - 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, - 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, - 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6b, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, + 0x74, 0x62, 0x61, 0x6b, 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, + 0x65, 0x6e, 0x2f, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( From 398db34329a11da9c676c977a93eb6bcb2fc63e3 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Thu, 9 Jul 2026 19:16:11 -0700 Subject: [PATCH 11/25] Revert "fix getcontainerconfigtemplate" This reverts commit bd65dc6e15ec9067527315dcc352cf21828348a9. --- aks-node-controller/parser/helper.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 655d7ad39bc..41fc83c7cd9 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -197,7 +197,7 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat // Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images/runtime) // compared to containerd 1.x (io.containerd.grpc.v1.cri). var _template *template.Template - if aksnodeconfig.GetContainerdConfig().GetUseContainerdV2() { + if isContainerdV2(aksnodeconfig.GetContainerdConfig().GetContainerdVersion()) { _template = containerdV2ConfigTemplate if noGPU { _template = containerdV2ConfigNoGPUTemplate @@ -217,6 +217,16 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat return buffer.String(), nil } +// isContainerdV2 returns true if the containerd version string indicates a 2.x release. +// Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images and +// io.containerd.cri.v1.runtime) compared to 1.x (io.containerd.grpc.v1.cri). +func isContainerdV2(version string) bool { + if version == "" { + return false + } + return helpers.IsKubernetesVersionGe(version, "2.0.0") +} + func getIsMIGNode(gpuInstanceProfile string) bool { return gpuInstanceProfile != "" } From f6aee27b00891d09e0bbb10bc25bc4e1465304a6 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Thu, 9 Jul 2026 19:40:44 -0700 Subject: [PATCH 12/25] use containerd --version to get containerd version --- aks-node-controller/parser/helper_test.go | 3 -- aks-node-controller/parser/parser.go | 38 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 83e71c1c529..09dc3bc59e7 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -522,7 +522,6 @@ func Test_getContainerdConfigV2(t *testing.T) { aksnodeconfig: &aksnodeconfigv1.Configuration{ ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdVersion: "2.0.0", - UseContainerdV2: true, }, }, }, @@ -556,7 +555,6 @@ oom_score = -999 NeedsCgroupv2: to.Ptr(true), ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdVersion: "2.0.1", - UseContainerdV2: true, }, GpuConfig: &aksnodeconfigv1.GpuConfig{ EnableNvidia: to.Ptr(true), @@ -593,7 +591,6 @@ oom_score = -999 aksnodeconfig: &aksnodeconfigv1.Configuration{ ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdVersion: "2.0.0", - UseContainerdV2: true, }, GpuConfig: &aksnodeconfigv1.GpuConfig{ EnableNvidia: to.Ptr(true), diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index 0eebd3e542b..d2ada966bf0 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -290,6 +290,18 @@ func mapToEnviron(input map[string]string) []string { } func BuildCSECmd(ctx context.Context, config *aksnodeconfigv1.Configuration) (*exec.Cmd, error) { + // Detect containerd version from the system if not already set in the config. + // This allows the correct containerd config template (v1 vs v2) to be selected + // on VHDs where the caller doesn't provide the version explicitly. + if config.GetContainerdConfig().GetContainerdVersion() == "" { + if version, err := detectContainerdVersion(ctx); err == nil && version != "" { + if config.ContainerdConfig == nil { + config.ContainerdConfig = &aksnodeconfigv1.ContainerdConfig{} + } + config.ContainerdConfig.ContainerdVersion = version + } + } + triggerBootstrapScript, err := executeBootstrapTemplate(config) if err != nil { return nil, fmt.Errorf("failed to execute the template: %w", err) @@ -302,3 +314,29 @@ func BuildCSECmd(ctx context.Context, config *aksnodeconfigv1.Configuration) (*e sort.Strings(cmd.Env) return cmd, nil } + +// detectContainerdVersion runs "containerd --version" and parses the version string. +// The expected output format is: "containerd v.. " +// e.g. "containerd containerd.io 1.7.22 c814c75..." or "containerd github.com/containerd/containerd/v2 v2.0.0 ..." +// Returns the semver version without the leading "v" prefix, or empty string if detection fails. +func detectContainerdVersion(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, "containerd", "--version").Output() + if err != nil { + return "", fmt.Errorf("running containerd --version: %w", err) + } + return parseContainerdVersionOutput(string(out)), nil +} + +// parseContainerdVersionOutput extracts the semver version from containerd --version output. +func parseContainerdVersionOutput(output string) string { + // Output format: "containerd v " + // Find the field that looks like a version (starts with a digit or "v" followed by a digit). + fields := strings.Fields(strings.TrimSpace(output)) + for _, field := range fields { + clean := strings.TrimPrefix(field, "v") + if len(clean) > 0 && clean[0] >= '0' && clean[0] <= '9' && strings.Contains(clean, ".") { + return clean + } + } + return "" +} From 9b82a4bba6a66a0078a37cb96c5fe360951e8494 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Thu, 9 Jul 2026 20:01:36 -0700 Subject: [PATCH 13/25] do not set containerd version for e2e --- e2e/node_config.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/e2e/node_config.go b/e2e/node_config.go index 87333eda500..3472d383501 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -265,7 +265,6 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod KubernetesVersion: cs.Properties.OrchestratorProfile.OrchestratorVersion, ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdDownloadUrlBase: nbc.CloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, - ContainerdVersion: getContainerdVersionFromDistro(nbc.AgentPoolProfile), }, OutboundCommand: helpers.GetDefaultOutboundCommand(), KubernetesCaCert: base64.StdEncoding.EncodeToString([]byte(cs.Properties.CertificateProfile.CaCertificate)), @@ -1088,14 +1087,3 @@ func pruneKubeletConfig(kubernetesVersion string, datamodel *datamodel.NodeBoots } return datamodel, nil } - -// getContainerdVersionFromDistro returns a containerd version string based on the distro, -// matching baker.go's logic where Is2404VHDDistro() selects the v2 containerd config template. -// This allows the aks-node-controller to select the correct containerd config template (v1 vs v2) -// based on the containerd version string. -func getContainerdVersionFromDistro(profile *datamodel.AgentPoolProfile) string { - if profile != nil && profile.Is2404VHDDistro() { - return "2.0.0" - } - return "" -} From f7c8f0b3a55d2a4fe45bb5bd744922cb10b73580 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Fri, 10 Jul 2026 16:02:14 -0700 Subject: [PATCH 14/25] fix version parsing and add UTs --- aks-node-controller/parser/parser.go | 9 +- aks-node-controller/parser/parser_test.go | 103 ++++++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index d2ada966bf0..93a10335399 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -329,12 +329,19 @@ func detectContainerdVersion(ctx context.Context) (string, error) { // parseContainerdVersionOutput extracts the semver version from containerd --version output. func parseContainerdVersionOutput(output string) string { - // Output format: "containerd v " + // Output format: "containerd " + // e.g. "containerd github.com/containerd/containerd/v2 2.3.2-1 fff62f1..." // Find the field that looks like a version (starts with a digit or "v" followed by a digit). + // Strip any package revision suffix (e.g. "-1" in "2.3.2-1") to get a clean semver. fields := strings.Fields(strings.TrimSpace(output)) for _, field := range fields { clean := strings.TrimPrefix(field, "v") if len(clean) > 0 && clean[0] >= '0' && clean[0] <= '9' && strings.Contains(clean, ".") { + // Strip package revision suffix: keep only "major.minor.patch" + // e.g. "2.3.2-1" -> "2.3.2" + if idx := strings.LastIndex(clean, "-"); idx > 0 { + clean = clean[:idx] + } return clean } } diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index d43ece17486..9a0c590bc71 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -375,6 +375,7 @@ oom_score = -999 KubernetesVersion: tt.k8sVersion, ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ ContainerdDownloadUrlBase: "https://storage.googleapis.com/cri-containerd-release/", + ContainerdVersion: "1.7.22", }, OutboundCommand: helpers.GetDefaultOutboundCommand(), KubeletConfig: &aksnodeconfigv1.KubeletConfig{ @@ -577,3 +578,105 @@ func assertHasKeyWithValue[K comparable, V any](t *testing.T, m map[K]V, key K, assert.Contains(t, m, key, "expected map to contain key: %v", key) assert.Equal(t, value, m[key], "expected map to have key-value pair %s=%v", key, value) } + +func TestParseContainerdVersionOutput(t *testing.T) { + tests := []struct { + name string + output string + want string + }{ + { + name: "containerd v2 with package revision", + output: "containerd github.com/containerd/containerd/v2 2.3.2-1 fff62f14765df376e5fc36f5a8f8e795b5670f61", + want: "2.3.2", + }, + { + name: "containerd v2 without package revision", + output: "containerd github.com/containerd/containerd/v2 v2.0.0 abc123", + want: "2.0.0", + }, + { + name: "containerd v1 with package revision", + output: "containerd containerd.io 1.7.22-1 c814c75abc123", + want: "1.7.22", + }, + { + name: "containerd v1 without package revision", + output: "containerd containerd.io 1.7.22 c814c75abc123", + want: "1.7.22", + }, + { + name: "empty output", + output: "", + want: "", + }, + { + name: "unexpected format", + output: "not a valid output", + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseContainerdVersionOutput(tt.output) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestBuildCSECmd_DetectsContainerdV2FromSystem(t *testing.T) { + // Create a fake containerd binary that outputs a v2 version string. + tmpDir := t.TempDir() + fakeBin := tmpDir + "/containerd" + err := os.WriteFile(fakeBin, []byte("#!/bin/sh\necho 'containerd github.com/containerd/containerd/v2 2.3.2-1 fff62f14765df376e5fc36f5a8f8e795b5670f61'\n"), 0755) + require.NoError(t, err) + + // Prepend tmpDir to PATH so our fake binary is found first. + t.Setenv("PATH", tmpDir+":"+os.Getenv("PATH")) + + config := &aksnodeconfigv1.Configuration{ + NeedsCgroupv2: to.Ptr(true), + // ContainerdVersion is intentionally NOT set — should be auto-detected. + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{}, + } + + cmd, err := BuildCSECmd(context.TODO(), config) + require.NoError(t, err) + + vars := environToMap(cmd.Env) + + // Verify the detected version was populated. + assert.Equal(t, "2.3.2", vars["CONTAINERD_VERSION"]) + + // Verify the v2 containerd config template was used (uses "io.containerd.cri.v1.images" path). + containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"])) + require.NoError(t, err) + assert.Contains(t, containerdConfig, `plugins."io.containerd.cri.v1.images"`) + assert.NotContains(t, containerdConfig, `plugins."io.containerd.grpc.v1.cri"`) +} + +func TestBuildCSECmd_FallsBackToV1WhenContainerdDetectionFails(t *testing.T) { + // Ensure no containerd binary is found by setting PATH to an empty temp dir. + tmpDir := t.TempDir() + t.Setenv("PATH", tmpDir) + + config := &aksnodeconfigv1.Configuration{ + // ContainerdVersion is intentionally NOT set and detection will fail. + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{}, + } + + // BuildCSECmd should NOT return an error even when containerd detection fails. + cmd, err := BuildCSECmd(context.TODO(), config) + require.NoError(t, err) + + vars := environToMap(cmd.Env) + + // Version should remain empty (detection failed gracefully). + assert.Equal(t, "", vars["CONTAINERD_VERSION"]) + + // Verify the v1 containerd config template was used (uses "io.containerd.grpc.v1.cri" path). + containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"])) + require.NoError(t, err) + assert.Contains(t, containerdConfig, `plugins."io.containerd.grpc.v1.cri"`) + assert.NotContains(t, containerdConfig, `plugins."io.containerd.cri.v1.images"`) +} From cd436d548b60b542550caf41cc2b826d85703df5 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Mon, 13 Jul 2026 12:33:43 -0700 Subject: [PATCH 15/25] do not set version in config --- aks-node-controller/parser/helper.go | 46 +++++++++++++++--- aks-node-controller/parser/helper_test.go | 7 +-- aks-node-controller/parser/parser.go | 59 +++++------------------ aks-node-controller/parser/parser_test.go | 3 -- 4 files changed, 56 insertions(+), 59 deletions(-) diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 41fc83c7cd9..e6f4e58665c 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -24,6 +24,7 @@ import ( "encoding/json" "fmt" "log" + "os/exec" "sort" "strconv" "strings" @@ -161,12 +162,12 @@ func getKubenetTemplate() string { } // getContainerdConfigBase64 returns the base64 encoded containerd config depending on whether the node is with GPU or not. -func getContainerdConfigBase64(aksnodeconfig *aksnodeconfigv1.Configuration) string { +func getContainerdConfigBase64(aksnodeconfig *aksnodeconfigv1.Configuration, containerdVersion string) string { if aksnodeconfig == nil { return "" } - containerdConfig, err := containerdConfigFromAKSNodeConfig(aksnodeconfig, false) + containerdConfig, err := containerdConfigFromAKSNodeConfig(aksnodeconfig, false, containerdVersion) if err != nil { return fmt.Sprintf("error getting containerd config from node bootstrap variables: %v", err) } @@ -175,12 +176,12 @@ func getContainerdConfigBase64(aksnodeconfig *aksnodeconfigv1.Configuration) str } // getNoGPUContainerdConfigBase64 returns the base64 encoded containerd config depending on whether the node is with GPU or not. -func getNoGPUContainerdConfigBase64(aksnodeconfig *aksnodeconfigv1.Configuration) string { +func getNoGPUContainerdConfigBase64(aksnodeconfig *aksnodeconfigv1.Configuration, containerdVersion string) string { if aksnodeconfig == nil { return "" } - containerdConfig, err := containerdConfigFromAKSNodeConfig(aksnodeconfig, true) + containerdConfig, err := containerdConfigFromAKSNodeConfig(aksnodeconfig, true, containerdVersion) if err != nil { return fmt.Sprintf("error getting No GPU containerd config from node bootstrap variables: %v", err) } @@ -188,7 +189,7 @@ func getNoGPUContainerdConfigBase64(aksnodeconfig *aksnodeconfigv1.Configuration return base64.StdEncoding.EncodeToString([]byte(containerdConfig)) } -func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, noGPU bool) (string, error) { +func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configuration, noGPU bool, containerdVersion string) (string, error) { if aksnodeconfig == nil { return "", fmt.Errorf("AKSNodeConfig is nil") } @@ -197,7 +198,7 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat // Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images/runtime) // compared to containerd 1.x (io.containerd.grpc.v1.cri). var _template *template.Template - if isContainerdV2(aksnodeconfig.GetContainerdConfig().GetContainerdVersion()) { + if isContainerdV2(containerdVersion) { _template = containerdV2ConfigTemplate if noGPU { _template = containerdV2ConfigNoGPUTemplate @@ -217,6 +218,18 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat return buffer.String(), nil } +// detectContainerdVersion runs "containerd --version" and parses the version string. +// The expected output format is: "containerd " +// e.g. "containerd containerd.io 1.7.22 c814c75..." or "containerd github.com/containerd/containerd/v2 v2.0.0 ..." +// Returns the semver version without the leading "v" prefix, or empty string if detection fails. +func detectContainerdVersion() (string, error) { + out, err := exec.Command("containerd", "--version").Output() + if err != nil { + return "", fmt.Errorf("running containerd --version: %w", err) + } + return parseContainerdVersionOutput(string(out)), nil +} + // isContainerdV2 returns true if the containerd version string indicates a 2.x release. // Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images and // io.containerd.cri.v1.runtime) compared to 1.x (io.containerd.grpc.v1.cri). @@ -765,6 +778,27 @@ func removeNewlines(str string) string { return sanitizedStr } +// parseContainerdVersionOutput extracts the semver version from containerd --version output. +func parseContainerdVersionOutput(output string) string { + // Output format: "containerd " + // e.g. "containerd github.com/containerd/containerd/v2 2.3.2-1 fff62f1..." + // Find the field that looks like a version (starts with a digit or "v" followed by a digit). + // Strip any package revision suffix (e.g. "-1" in "2.3.2-1") to get a clean semver. + fields := strings.Fields(strings.TrimSpace(output)) + for _, field := range fields { + clean := strings.TrimPrefix(field, "v") + if len(clean) > 0 && clean[0] >= '0' && clean[0] <= '9' && strings.Contains(clean, ".") { + // Strip package revision suffix: keep only "major.minor.patch" + // e.g. "2.3.2-1" -> "2.3.2" + if idx := strings.LastIndex(clean, "-"); idx > 0 { + clean = clean[:idx] + } + return clean + } + } + return "" +} + // ---------------------- Start of localdns related helper code ----------------------// // FuncMap used for generating localdns corefile. diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 09dc3bc59e7..feb623dce91 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -499,7 +499,7 @@ oom_score = -999 } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := getContainerdConfigBase64(tt.args.aksnodeconfig); got != tt.want { + if got := getContainerdConfigBase64(tt.args.aksnodeconfig, ""); got != tt.want { t.Errorf("getContainerdConfig() = %v, want %v", got, tt.want) } }) @@ -655,11 +655,12 @@ oom_score = -999 } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + containerdVersion := tt.args.aksnodeconfig.GetContainerdConfig().GetContainerdVersion() var got string if tt.args.noGpu { - got = getNoGPUContainerdConfigBase64(tt.args.aksnodeconfig) + got = getNoGPUContainerdConfigBase64(tt.args.aksnodeconfig, containerdVersion) } else { - got = getContainerdConfigBase64(tt.args.aksnodeconfig) + got = getContainerdConfigBase64(tt.args.aksnodeconfig, containerdVersion) } if got != tt.want { t.Errorf("getContainerdConfig() = %v, want %v", got, tt.want) diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index 93a10335399..6cc4679d52c 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -31,6 +31,16 @@ func executeBootstrapTemplate(inputContract *aksnodeconfigv1.Configuration) (str //nolint:funlen func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string { + // Detect containerd version from the system if not already set in the config. + // This allows the correct containerd config template (v1 vs v2) to be selected + // on VHDs where the caller doesn't provide the version explicitly. + var containerdVersion string + if config.GetContainerdConfig().GetContainerdVersion() == "" { + if version, err := detectContainerdVersion(); err == nil && version != "" { + containerdVersion = version + } + } + cloudProviderSettings := getCloudProviderSettings(config) env := map[string]string{ "PROVISION_OUTPUT": "/var/log/azure/cluster-provision-cse-output.log", @@ -167,8 +177,8 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string { "AZURE_ENVIRONMENT_FILEPATH": getAzureEnvironmentFilepath(config), "KUBE_CA_CRT": config.GetKubernetesCaCert(), "KUBENET_TEMPLATE": getKubenetTemplate(), - "CONTAINERD_CONFIG_CONTENT": getContainerdConfigBase64(config), - "CONTAINERD_CONFIG_NO_GPU_CONTENT": getNoGPUContainerdConfigBase64(config), + "CONTAINERD_CONFIG_CONTENT": getContainerdConfigBase64(config, containerdVersion), + "CONTAINERD_CONFIG_NO_GPU_CONTENT": getNoGPUContainerdConfigBase64(config, containerdVersion), "IS_KATA": fmt.Sprintf("%v", config.GetIsKata()), "ARTIFACT_STREAMING_ENABLED": fmt.Sprintf("%v", config.GetEnableArtifactStreaming()), "SYSCTL_CONTENT": getSysctlContent(config.GetCustomLinuxOsConfig().GetSysctlConfig()), @@ -290,18 +300,6 @@ func mapToEnviron(input map[string]string) []string { } func BuildCSECmd(ctx context.Context, config *aksnodeconfigv1.Configuration) (*exec.Cmd, error) { - // Detect containerd version from the system if not already set in the config. - // This allows the correct containerd config template (v1 vs v2) to be selected - // on VHDs where the caller doesn't provide the version explicitly. - if config.GetContainerdConfig().GetContainerdVersion() == "" { - if version, err := detectContainerdVersion(ctx); err == nil && version != "" { - if config.ContainerdConfig == nil { - config.ContainerdConfig = &aksnodeconfigv1.ContainerdConfig{} - } - config.ContainerdConfig.ContainerdVersion = version - } - } - triggerBootstrapScript, err := executeBootstrapTemplate(config) if err != nil { return nil, fmt.Errorf("failed to execute the template: %w", err) @@ -314,36 +312,3 @@ func BuildCSECmd(ctx context.Context, config *aksnodeconfigv1.Configuration) (*e sort.Strings(cmd.Env) return cmd, nil } - -// detectContainerdVersion runs "containerd --version" and parses the version string. -// The expected output format is: "containerd v.. " -// e.g. "containerd containerd.io 1.7.22 c814c75..." or "containerd github.com/containerd/containerd/v2 v2.0.0 ..." -// Returns the semver version without the leading "v" prefix, or empty string if detection fails. -func detectContainerdVersion(ctx context.Context) (string, error) { - out, err := exec.CommandContext(ctx, "containerd", "--version").Output() - if err != nil { - return "", fmt.Errorf("running containerd --version: %w", err) - } - return parseContainerdVersionOutput(string(out)), nil -} - -// parseContainerdVersionOutput extracts the semver version from containerd --version output. -func parseContainerdVersionOutput(output string) string { - // Output format: "containerd " - // e.g. "containerd github.com/containerd/containerd/v2 2.3.2-1 fff62f1..." - // Find the field that looks like a version (starts with a digit or "v" followed by a digit). - // Strip any package revision suffix (e.g. "-1" in "2.3.2-1") to get a clean semver. - fields := strings.Fields(strings.TrimSpace(output)) - for _, field := range fields { - clean := strings.TrimPrefix(field, "v") - if len(clean) > 0 && clean[0] >= '0' && clean[0] <= '9' && strings.Contains(clean, ".") { - // Strip package revision suffix: keep only "major.minor.patch" - // e.g. "2.3.2-1" -> "2.3.2" - if idx := strings.LastIndex(clean, "-"); idx > 0 { - clean = clean[:idx] - } - return clean - } - } - return "" -} diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index 9a0c590bc71..fd44fd87744 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -645,9 +645,6 @@ func TestBuildCSECmd_DetectsContainerdV2FromSystem(t *testing.T) { vars := environToMap(cmd.Env) - // Verify the detected version was populated. - assert.Equal(t, "2.3.2", vars["CONTAINERD_VERSION"]) - // Verify the v2 containerd config template was used (uses "io.containerd.cri.v1.images" path). containerdConfig, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_NO_GPU_CONTENT"])) require.NoError(t, err) From fb3686bc7ae32ab31a45fa56485c800232aca153 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Mon, 13 Jul 2026 15:29:43 -0700 Subject: [PATCH 16/25] account for prerelease version --- aks-node-controller/parser/helper.go | 8 ++++---- aks-node-controller/parser/parser_test.go | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index e6f4e58665c..91c9fac8448 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -783,14 +783,14 @@ func parseContainerdVersionOutput(output string) string { // Output format: "containerd " // e.g. "containerd github.com/containerd/containerd/v2 2.3.2-1 fff62f1..." // Find the field that looks like a version (starts with a digit or "v" followed by a digit). - // Strip any package revision suffix (e.g. "-1" in "2.3.2-1") to get a clean semver. + // Strip any package revision or pre-release suffix to get a clean major.minor.patch. fields := strings.Fields(strings.TrimSpace(output)) for _, field := range fields { clean := strings.TrimPrefix(field, "v") if len(clean) > 0 && clean[0] >= '0' && clean[0] <= '9' && strings.Contains(clean, ".") { - // Strip package revision suffix: keep only "major.minor.patch" - // e.g. "2.3.2-1" -> "2.3.2" - if idx := strings.LastIndex(clean, "-"); idx > 0 { + // Strip everything after the first "-" (package revision or pre-release suffix). + // e.g. "2.3.2-1" -> "2.3.2", "2.0.0-beta.1" -> "2.0.0" + if idx := strings.Index(clean, "-"); idx > 0 { clean = clean[:idx] } return clean diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index fd44fd87744..bcb5e2d2602 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -605,6 +605,16 @@ func TestParseContainerdVersionOutput(t *testing.T) { output: "containerd containerd.io 1.7.22 c814c75abc123", want: "1.7.22", }, + { + name: "containerd v2 pre-release suffix", + output: "containerd github.com/containerd/containerd/v2 2.0.0-beta.1 abc123", + want: "2.0.0", + }, + { + name: "containerd v2 rc suffix", + output: "containerd github.com/containerd/containerd/v2 v2.1.0-rc.2 abc123", + want: "2.1.0", + }, { name: "empty output", output: "", From b644e92b4b15aca24b92bb6d19c538580bf1d950 Mon Sep 17 00:00:00 2001 From: lilypan26 Date: Mon, 13 Jul 2026 16:18:41 -0700 Subject: [PATCH 17/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- aks-node-controller/parser/parser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index e118cf63e7b..678bb6b41fc 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -34,8 +34,8 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string { // Detect containerd version from the system if not already set in the config. // This allows the correct containerd config template (v1 vs v2) to be selected // on VHDs where the caller doesn't provide the version explicitly. - var containerdVersion string - if config.GetContainerdConfig().GetContainerdVersion() == "" { + containerdVersion := config.GetContainerdConfig().GetContainerdVersion() + if containerdVersion == "" { if version, err := detectContainerdVersion(); err == nil && version != "" { containerdVersion = version } From ac539d2bd5c498af98f00e9c32c091a1da9688ff Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Mon, 13 Jul 2026 17:29:08 -0700 Subject: [PATCH 18/25] use commandcontext --- aks-node-controller/parser/helper.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 385311ad17f..934021f7e04 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -19,6 +19,7 @@ package parser import ( "bytes" + "context" _ "embed" "encoding/base64" "encoding/json" @@ -224,7 +225,7 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat // e.g. "containerd containerd.io 1.7.22 c814c75..." or "containerd github.com/containerd/containerd/v2 v2.0.0 ..." // Returns the semver version without the leading "v" prefix, or empty string if detection fails. func detectContainerdVersion() (string, error) { - out, err := exec.Command("containerd", "--version").Output() + out, err := exec.CommandContext(context.Background(), "containerd", "--version").Output() if err != nil { return "", fmt.Errorf("running containerd --version: %w", err) } From 450a7139e84b33a089273454d3f9e63eb0473e09 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Tue, 14 Jul 2026 14:35:53 -0700 Subject: [PATCH 19/25] use correct containerdconfig for azurelinux --- pkg/agent/baker.go | 4 ++-- pkg/agent/datamodel/sig_config.go | 14 ++++++++++++++ pkg/agent/datamodel/types.go | 8 ++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 8f08d57e0d4..108cbda2925 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -1014,7 +1014,7 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration }, "GetContainerdConfigContent": func() string { output, err := containerdConfigFromTemplate(config, profile, func(profile *datamodel.AgentPoolProfile) ContainerdConfigTemplate { - if profile.Is2404VHDDistro() { + if profile.IsContainerdV2Distro() { return containerdV2ConfigTemplate } return containerdV1ConfigTemplate @@ -1026,7 +1026,7 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration }, "GetContainerdConfigNoGPUContent": func() string { output, err := containerdConfigFromTemplate(config, profile, func(profile *datamodel.AgentPoolProfile) ContainerdConfigTemplate { - if profile.Is2404VHDDistro() { + if profile.IsContainerdV2Distro() { return containerdV2NoGPUConfigTemplate } return containerdV1NoGPUConfigTemplate diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 6d7156b7a45..62591113ae8 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -264,6 +264,20 @@ var AvailableAzureLinuxCgroupV2Distros = []Distro{ AKSAzureLinuxV3OSGuardGen2FIPSTL, } +//nolint:gochecknoglobals +var AvailableAzureLinuxV3Distros = []Distro{ + AKSAzureLinuxV3, + AKSAzureLinuxV3Gen2, + AKSAzureLinuxV3FIPS, + AKSAzureLinuxV3Gen2FIPS, + AKSAzureLinuxV3Gen2Kata, + AKSAzureLinuxV3Arm64Gen2, + AKSAzureLinuxV3Arm64Gen2FIPS, + AKSAzureLinuxV3Gen2TL, + AKSAzureLinuxV3CVMGen2, + AKSAzureLinuxV3OSGuardGen2FIPSTL, +} + //nolint:gochecknoglobals var AvailableAzureLinuxOSGuardDistros = []Distro{ AKSAzureLinuxV3OSGuardGen2FIPSTL, diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index 5c4dba047a8..796db922b01 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -330,6 +330,10 @@ func (d Distro) IsAzureLinuxOSGuardDistro() bool { return slices.Contains(AvailableAzureLinuxOSGuardDistros, d) } +func (d Distro) IsAzureLinuxV3Distro() bool { + return slices.Contains(AvailableAzureLinuxV3Distros, d) +} + /* KeyvaultSecretRef specifies path to the Azure keyvault along with secret name and (optionaly) version for Service Principal's secret. @@ -1184,6 +1188,10 @@ func (a *AgentPoolProfile) Is2604VHDDistro() bool { return a.Distro.Is2604VHDDistro() } +func (a *AgentPoolProfile) IsContainerdV2Distro() bool { + return a.Distro.IsACLDistro() || a.Distro.IsAzureLinuxV3Distro() +} + // IsAzureLinuxCgroupV2VHDDistro returns true if the distro uses Azure Linux CgrpupV2 VHD. func (a *AgentPoolProfile) IsAzureLinuxCgroupV2VHDDistro() bool { return a.Distro.IsAzureLinuxCgroupV2VHDDistro() From 94b644e75bed74958ff6cf3469092dbb51732802 Mon Sep 17 00:00:00 2001 From: lilypan26 Date: Tue, 14 Jul 2026 14:49:02 -0700 Subject: [PATCH 20/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pkg/agent/datamodel/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index 796db922b01..ca978dd4f35 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1189,7 +1189,7 @@ func (a *AgentPoolProfile) Is2604VHDDistro() bool { } func (a *AgentPoolProfile) IsContainerdV2Distro() bool { - return a.Distro.IsACLDistro() || a.Distro.IsAzureLinuxV3Distro() + return a.Distro.Is2404VHDDistro() || a.Distro.IsACLDistro() || a.Distro.IsAzureLinuxV3Distro() } // IsAzureLinuxCgroupV2VHDDistro returns true if the distro uses Azure Linux CgrpupV2 VHD. From 0e3804b2d6141be90654945caa2c2f93281b4669 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Wed, 15 Jul 2026 11:50:26 -0700 Subject: [PATCH 21/25] fix tests and address comments --- aks-node-controller/app.go | 6 +-- aks-node-controller/parser/helper.go | 44 ++++++++++++------- aks-node-controller/parser/helper_test.go | 2 - aks-node-controller/parser/parser.go | 6 +-- .../parser/templates/containerd_v2.toml.gtpl | 1 - pkg/agent/datamodel/sig_config.go | 2 + 6 files changed, 37 insertions(+), 24 deletions(-) diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index d2f20545d2e..88f4684ebb5 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -356,14 +356,14 @@ func diffEnvMaps(pcEnv, nbcEnv map[string]string) []string { nbcVal, inNBC := nbcEnv[key] switch { case inPC && !inNBC: - diffs = append(diffs, fmt.Sprintf("only-in-pc: %s", key)) + diffs = append(diffs, fmt.Sprintf("only-in-pc: %s=%s", key, pcVal)) case !inPC && inNBC: if !isExpectedDiffCSEVar(key) { - diffs = append(diffs, fmt.Sprintf("only-in-nbc: %s", key)) + diffs = append(diffs, fmt.Sprintf("only-in-nbc: %s=%s", key, nbcVal)) } case !envValsEqualForKey(key, pcVal, nbcVal): if !isExpectedDiffCSEVar(key) { - diffs = append(diffs, fmt.Sprintf("differs: %s", key)) + diffs = append(diffs, fmt.Sprintf("differs: %s pc=%s nbc=%s", key, pcVal, nbcVal)) } } } diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 6d083ee50d6..c07d4610282 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -232,8 +232,8 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat // The expected output format is: "containerd " // e.g. "containerd containerd.io 1.7.22 c814c75..." or "containerd github.com/containerd/containerd/v2 v2.0.0 ..." // Returns the semver version without the leading "v" prefix, or empty string if detection fails. -func detectContainerdVersion() (string, error) { - out, err := exec.CommandContext(context.Background(), "containerd", "--version").Output() +func detectContainerdVersion(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, "containerd", "--version").Output() if err != nil { return "", fmt.Errorf("running containerd --version: %w", err) } @@ -793,24 +793,38 @@ func removeNewlines(str string) string { } // parseContainerdVersionOutput extracts the semver version from containerd --version output. +// The output format is: "containerd " +// e.g. "containerd containerd.io 1.7.22 c814c75..." or "containerd github.com/containerd/containerd/v2 2.0.0 ..." +// The version (3rd field) could be in the format "1.6.24-11-ubuntu1~24.04.1" or "2.0.0-6.azl3" or just "2.0.0", +// we extract the major.minor.patch version only. func parseContainerdVersionOutput(output string) string { - // Output format: "containerd " - // e.g. "containerd github.com/containerd/containerd/v2 2.3.2-1 fff62f1..." - // Find the field that looks like a version (starts with a digit or "v" followed by a digit). - // Strip any package revision or pre-release suffix to get a clean major.minor.patch. fields := strings.Fields(strings.TrimSpace(output)) - for _, field := range fields { - clean := strings.TrimPrefix(field, "v") - if len(clean) > 0 && clean[0] >= '0' && clean[0] <= '9' && strings.Contains(clean, ".") { - // Strip everything after the first "-" (package revision or pre-release suffix). - // e.g. "2.3.2-1" -> "2.3.2", "2.0.0-beta.1" -> "2.0.0" - if idx := strings.Index(clean, "-"); idx > 0 { - clean = clean[:idx] + if len(fields) < 3 { + return "" + } + // Take the 3rd field and strip any leading "v" prefix. + version := strings.TrimPrefix(fields[2], "v") + // Strip everything after the first "-" (package revision or pre-release suffix). + // e.g. "2.3.2-1" -> "2.3.2", "2.0.0-beta.1" -> "2.0.0" + if idx := strings.Index(version, "-"); idx > 0 { + version = version[:idx] + } + // Validate the result is a valid major.minor.patch version. + parts := strings.Split(version, ".") + if len(parts) != 3 { + return "" + } + for _, p := range parts { + if len(p) == 0 { + return "" + } + for _, c := range p { + if c < '0' || c > '9' { + return "" } - return clean } } - return "" + return version } // ---------------------- Start of localdns related helper code ----------------------// diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 4ac17d45ebf..2a53afad285 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -545,7 +545,6 @@ oom_score = -999 BinaryName = "/usr/bin/runc" [metrics] address = "0.0.0.0:10257" - `)), }, { @@ -582,7 +581,6 @@ oom_score = -999 BinaryName = "/usr/bin/nvidia-container-runtime" [metrics] address = "0.0.0.0:10257" - `)), }, { diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index 678bb6b41fc..3c54814420c 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -30,13 +30,13 @@ func executeBootstrapTemplate(inputContract *aksnodeconfigv1.Configuration) (str } //nolint:funlen -func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string { +func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration) map[string]string { // Detect containerd version from the system if not already set in the config. // This allows the correct containerd config template (v1 vs v2) to be selected // on VHDs where the caller doesn't provide the version explicitly. containerdVersion := config.GetContainerdConfig().GetContainerdVersion() if containerdVersion == "" { - if version, err := detectContainerdVersion(); err == nil && version != "" { + if version, err := detectContainerdVersion(ctx); err == nil && version != "" { containerdVersion = version } } @@ -306,7 +306,7 @@ func BuildCSECmd(ctx context.Context, config *aksnodeconfigv1.Configuration) (*e // Convert to one-liner triggerBootstrapScript = strings.ReplaceAll(triggerBootstrapScript, "\n", " ") cmd := exec.CommandContext(ctx, "/bin/bash", "-c", triggerBootstrapScript) - env := mapToEnviron(getCSEEnv(config)) + env := mapToEnviron(getCSEEnv(ctx, config)) cmd.Env = append(os.Environ(), env...) // append existing environment variables sort.Strings(cmd.Env) return cmd, nil diff --git a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl index 6515519bd6e..fd11c508145 100644 --- a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl @@ -70,4 +70,3 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options] ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml" {{- end}} - diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 03db1a7d56b..8cff7e62237 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -289,6 +289,8 @@ var AvailableAzureLinuxV3Distros = []Distro{ AKSAzureLinuxV3Gen2TL, AKSAzureLinuxV3CVMGen2, AKSAzureLinuxV3OSGuardGen2FIPSTL, + AKSAzureLinuxV3EdgeZone, + AKSAzureLinuxV3EdgeZoneGen2, } //nolint:gochecknoglobals From cbe3c84091dd7e7c326f1de1bbc15dadd73b72ac Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Wed, 15 Jul 2026 13:15:30 -0700 Subject: [PATCH 22/25] revert logging value --- aks-node-controller/app.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 88f4684ebb5..d2f20545d2e 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -356,14 +356,14 @@ func diffEnvMaps(pcEnv, nbcEnv map[string]string) []string { nbcVal, inNBC := nbcEnv[key] switch { case inPC && !inNBC: - diffs = append(diffs, fmt.Sprintf("only-in-pc: %s=%s", key, pcVal)) + diffs = append(diffs, fmt.Sprintf("only-in-pc: %s", key)) case !inPC && inNBC: if !isExpectedDiffCSEVar(key) { - diffs = append(diffs, fmt.Sprintf("only-in-nbc: %s=%s", key, nbcVal)) + diffs = append(diffs, fmt.Sprintf("only-in-nbc: %s", key)) } case !envValsEqualForKey(key, pcVal, nbcVal): if !isExpectedDiffCSEVar(key) { - diffs = append(diffs, fmt.Sprintf("differs: %s pc=%s nbc=%s", key, pcVal, nbcVal)) + diffs = append(diffs, fmt.Sprintf("differs: %s", key)) } } } From daccbe324c469878271b332ade7c1bdc54d4646c Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Wed, 15 Jul 2026 14:13:28 -0700 Subject: [PATCH 23/25] always get containerd version --- aks-node-controller/parser/parser.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index 3c54814420c..2547f834784 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -31,16 +31,7 @@ func executeBootstrapTemplate(inputContract *aksnodeconfigv1.Configuration) (str //nolint:funlen func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration) map[string]string { - // Detect containerd version from the system if not already set in the config. - // This allows the correct containerd config template (v1 vs v2) to be selected - // on VHDs where the caller doesn't provide the version explicitly. - containerdVersion := config.GetContainerdConfig().GetContainerdVersion() - if containerdVersion == "" { - if version, err := detectContainerdVersion(ctx); err == nil && version != "" { - containerdVersion = version - } - } - + containerdVersion, _ := detectContainerdVersion(ctx) cloudProviderSettings := getCloudProviderSettings(config) env := map[string]string{ "PROVISION_OUTPUT": "/var/log/azure/cluster-provision-cse-output.log", From 25367a96fb30ebc5eae8216129225ce9eb3a79b1 Mon Sep 17 00:00:00 2001 From: Lily Pan Date: Wed, 15 Jul 2026 14:42:57 -0700 Subject: [PATCH 24/25] Fix UTs --- aks-node-controller/parser/parser_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index 4fcd8d37e49..c1e26bd53c5 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -266,6 +266,12 @@ oom_score = -999 for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // Set up a fake containerd binary returning v1 so detectContainerdVersion + // consistently returns 1.7.22 regardless of the host system. + fakeBinDir := t.TempDir() + require.NoError(t, os.WriteFile(fakeBinDir+"/containerd", []byte("#!/bin/sh\necho 'containerd containerd.io 1.7.22 c814c75'\n"), 0755)) + t.Setenv("PATH", fakeBinDir+":"+os.Getenv("PATH")) + cs := &datamodel.ContainerService{ Location: "southcentralus", Type: "Microsoft.ContainerService/ManagedClusters", From 857e4a7db4c99322d4aa9acee5ec2d718123ac90 Mon Sep 17 00:00:00 2001 From: lilypan26 Date: Wed, 15 Jul 2026 14:51:47 -0700 Subject: [PATCH 25/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pkg/agent/datamodel/types.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index 999ac6a6866..6b49358f7d2 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1197,6 +1197,9 @@ func (a *AgentPoolProfile) Is2604VHDDistro() bool { } func (a *AgentPoolProfile) IsContainerdV2Distro() bool { + if a.Distro.IsKataDistro() { + return false + } return a.Distro.Is2404VHDDistro() || a.Distro.IsACLDistro() || a.Distro.IsAzureLinuxV3Distro() }