Skip to content

feat: add scriptless containerd 2.0 support #8825

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
lilypan26 wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: add scriptless containerd 2.0 support #8825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 36 additions & 5 deletions aks-node-controller/parser/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,18 @@ var (
containerdConfigNoGPUTemplate = template.Must(
template.New("nogpucontainerdconfig").Funcs(getFuncMapForContainerdConfigTemplate()).Parse(containerdConfigNoGPUTemplateText),
)
//go:embed templates/containerd_v2.toml.gtpl
containerdV2ConfigTemplateText string
//nolint:gochecknoglobals
containerdV2ConfigTemplate = template.Must(
template.New("containerdv2config").Funcs(getFuncMapForContainerdConfigTemplate()).Parse(containerdV2ConfigTemplateText),
)
//go:embed templates/containerd_v2_no_GPU.toml.gtpl
containerdV2ConfigNoGPUTemplateText string
//nolint:gochecknoglobals
containerdV2ConfigNoGPUTemplate = template.Must(
template.New("nogpucontainerdv2config").Funcs(getFuncMapForContainerdConfigTemplate()).Parse(containerdV2ConfigNoGPUTemplateText),
)

//go:embed templates/localdns.toml.gtpl
localDnsCorefileTemplateText string
Expand Down Expand Up @@ -181,11 +193,20 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat
return "", fmt.Errorf("AKSNodeConfig is nil")
}

// TODO: add containerdv2 support
// the containerd config template is different based on whether the node is with GPU or not.
_template := containerdConfigTemplate
if noGPU {
_template = containerdConfigNoGPUTemplate
// Select the appropriate containerd config template based on version and GPU presence.
// Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images/runtime)
// compared to containerd 1.x (io.containerd.grpc.v1.cri).
var _template *template.Template
if isContainerdV2(aksnodeconfig.GetContainerdConfig().GetContainerdVersion()) {
_template = containerdV2ConfigTemplate
if noGPU {
_template = containerdV2ConfigNoGPUTemplate
}
} else {
_template = containerdConfigTemplate
if noGPU {
_template = containerdConfigNoGPUTemplate
}
}

var buffer bytes.Buffer
Expand All @@ -196,6 +217,16 @@ func containerdConfigFromAKSNodeConfig(aksnodeconfig *aksnodeconfigv1.Configurat
return buffer.String(), nil
}

// isContainerdV2 returns true if the containerd version string indicates a 2.x release.
// Containerd 2.x uses different CRI plugin paths (io.containerd.cri.v1.images and
// io.containerd.cri.v1.runtime) compared to 1.x (io.containerd.grpc.v1.cri).
func isContainerdV2(version string) bool {
if version == "" {
return false
}
return helpers.IsKubernetesVersionGe(version, "2.0.0")
}

func getIsMIGNode(gpuInstanceProfile string) bool {
return gpuInstanceProfile != ""
}
Expand Down
185 changes: 185 additions & 0 deletions aks-node-controller/parser/helper_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,191 @@ oom_score = -999
}
}

func Test_isContainerdV2(t *testing.T) {
tests := []struct {
name string
version string
want bool
}{
{name: "empty version", version: "", want: false},
{name: "v1.6.x", version: "1.6.28", want: false},
{name: "v1.7.x", version: "1.7.22", want: false},
{name: "v2.0.0", version: "2.0.0", want: true},
{name: "v2.0.1", version: "2.0.1", want: true},
{name: "v2.1.0", version: "2.1.0", want: true},
{name: "invalid version", version: "notaversion", want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isContainerdV2(tt.version); got != tt.want {
t.Errorf("isContainerdV2(%q) = %v, want %v", tt.version, got, tt.want)
}
})
}
}

func Test_getContainerdConfigV2(t *testing.T) {
type args struct {
aksnodeconfig *aksnodeconfigv1.Configuration
noGpu bool
}
tests := []struct {
name string
args args
want string
}{
{
name: "Containerd v2 default config",
args: args{
aksnodeconfig: &aksnodeconfigv1.Configuration{
NeedsCgroupv2: to.Ptr(true),
ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{
ContainerdVersion: "2.0.0",
},
},
},
want: base64.StdEncoding.EncodeToString([]byte(`version = 2
oom_score = -999
[plugins."io.containerd.cri.v1.images"]
[plugins."io.containerd.cri.v1.images".pinned_images]
sandbox = ""
Comment on lines +552 to +556
[plugins."io.containerd.cri.v1.images".registry.headers]
X-Meta-Source-Client = ["azure/aks"]
[plugins."io.containerd.cri.v1.runtime".containerd]
default_runtime_name = "runc"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
SystemdCgroup = true
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/runc"
[metrics]
address = "0.0.0.0:10257"
`)),
},
{
name: "Containerd v2 with GPU",
args: args{
aksnodeconfig: &aksnodeconfigv1.Configuration{
NeedsCgroupv2: to.Ptr(true),
ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{
ContainerdVersion: "2.0.1",
},
GpuConfig: &aksnodeconfigv1.GpuConfig{
EnableNvidia: to.Ptr(true),
},
},
noGpu: false,
},
want: base64.StdEncoding.EncodeToString([]byte(`version = 2
oom_score = -999
[plugins."io.containerd.cri.v1.images"]
[plugins."io.containerd.cri.v1.images".pinned_images]
sandbox = ""
[plugins."io.containerd.cri.v1.images".registry.headers]
X-Meta-Source-Client = ["azure/aks"]
[plugins."io.containerd.cri.v1.runtime".containerd]
default_runtime_name = "nvidia-container-runtime"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options]
BinaryName = "/usr/bin/nvidia-container-runtime"
SystemdCgroup = true
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/nvidia-container-runtime"
[metrics]
address = "0.0.0.0:10257"
`)),
},
{
name: "Containerd v2 no GPU template",
args: args{
aksnodeconfig: &aksnodeconfigv1.Configuration{
NeedsCgroupv2: to.Ptr(true),
ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{
ContainerdVersion: "2.0.0",
},
GpuConfig: &aksnodeconfigv1.GpuConfig{
EnableNvidia: to.Ptr(true),
},
},
noGpu: true,
},
want: base64.StdEncoding.EncodeToString([]byte(`version = 2
oom_score = -999
[plugins."io.containerd.cri.v1.images"]
[plugins."io.containerd.cri.v1.images".pinned_images]
sandbox = ""
[plugins."io.containerd.cri.v1.images".registry.headers]
X-Meta-Source-Client = ["azure/aks"]
[plugins."io.containerd.cri.v1.runtime".containerd]
default_runtime_name = "runc"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
SystemdCgroup = true
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/runc"
[metrics]
address = "0.0.0.0:10257"
`)),
},
{
name: "Containerd v1 still uses old templates",
args: args{
aksnodeconfig: &aksnodeconfigv1.Configuration{
NeedsCgroupv2: to.Ptr(true),
ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{
ContainerdVersion: "1.7.22",
},
},
},
want: base64.StdEncoding.EncodeToString([]byte(`version = 2
oom_score = -999
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = ""
enable_cdi = true
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/runc"
[plugins."io.containerd.grpc.v1.cri".registry.headers]
X-Meta-Source-Client = ["azure/aks"]
[metrics]
address = "0.0.0.0:10257"
`)),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var got string
if tt.args.noGpu {
got = getNoGPUContainerdConfigBase64(tt.args.aksnodeconfig)
} else {
got = getContainerdConfigBase64(tt.args.aksnodeconfig)
}
if got != tt.want {
t.Errorf("getContainerdConfig() = %v, want %v", got, tt.want)
}
})
}
}

func Test_getKubenetTemplate(t *testing.T) {
tests := []struct {
name string
Expand Down
73 changes: 73 additions & 0 deletions aks-node-controller/parser/templates/containerd_v2.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
[plugins."io.containerd.cri.v1.images"]
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
disable_snapshot_annotations = false
{{- end}}
[plugins."io.containerd.cri.v1.images".pinned_images]
sandbox = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}"
{{- if isKubernetesVersionGe .GetKubernetesVersion "1.22.0"}}
[plugins."io.containerd.cri.v1.images".registry]
config_path = "/etc/containerd/certs.d"
{{- end}}
[plugins."io.containerd.cri.v1.images".registry.headers]
X-Meta-Source-Client = ["azure/aks"]
[plugins."io.containerd.cri.v1.runtime".containerd]
{{- if getEnableNvidia . }}
default_runtime_name = "nvidia-container-runtime"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia-container-runtime.options]
BinaryName = "/usr/bin/nvidia-container-runtime"
SystemdCgroup = true
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/nvidia-container-runtime"
{{- else}}
default_runtime_name = "runc"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
SystemdCgroup = true
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/runc"
{{- end}}
{{- if getEnsureNoDupePromiscuousBridge .GetNetworkConfig }}
[plugins."io.containerd.cri.v1.runtime".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = "/etc/containerd/kubenet_template.conf"
{{- end}}
[metrics]
address = "0.0.0.0:10257"
{{- if .GetEnableArtifactStreaming }}
[proxy_plugins]
[proxy_plugins.overlaybd]
type = "snapshot"
address = "/run/overlaybd-snapshotter/overlaybd.sock"
{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
address = "/run/containerd/tardev-snapshotter.sock"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc]
snapshotter = "tardev"
runtime_type = "io.containerd.kata-cc.v2"
privileged_without_host_devices = true
pod_annotations = ["io.katacontainers.*"]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-cc.options]
ConfigPath = "/opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml"
{{- end}}

52 changes: 52 additions & 0 deletions aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
[plugins."io.containerd.cri.v1.images"]
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
disable_snapshot_annotations = false
{{- end}}
[plugins."io.containerd.cri.v1.images".pinned_images]
sandbox = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}"
{{- if isKubernetesVersionGe .GetKubernetesVersion "1.22.0"}}
[plugins."io.containerd.cri.v1.images".registry]
config_path = "/etc/containerd/certs.d"
{{- end}}
[plugins."io.containerd.cri.v1.images".registry.headers]
X-Meta-Source-Client = ["azure/aks"]
[plugins."io.containerd.cri.v1.runtime".containerd]
default_runtime_name = "runc"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
BinaryName = "/usr/bin/runc"
SystemdCgroup = true
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.untrusted.options]
BinaryName = "/usr/bin/runc"
{{- if getEnsureNoDupePromiscuousBridge .GetNetworkConfig }}
[plugins."io.containerd.cri.v1.runtime".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = "/etc/containerd/kubenet_template.conf"
{{- end}}
[metrics]
address = "0.0.0.0:10257"
{{- if .GetEnableArtifactStreaming }}
[proxy_plugins]
[proxy_plugins.overlaybd]
type = "snapshot"
address = "/run/overlaybd-snapshotter/overlaybd.sock"
{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
address = "/run/containerd/tardev-snapshotter.sock"
{{- end}}
Loading
Loading