diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index a766f061f4..21418563c4 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -1492,6 +1492,73 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } } + imageSet, err := imageset.GetImageSet(ctx, r.client, instance.Spec.Variant) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Error getting ImageSet", err, reqLogger) + return reconcile.Result{}, err + } + + if imageSet == nil { + // There is no imageSet for the configured variant, but check to see if there are any + // ImageSets with a different variant so we can give the user some kind of indication + // to why an existing ImageSet is being ignored. + nvis, err := imageset.DoesNonVariantImageSetExist(ctx, r.client, instance.Spec.Variant) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Error checking for non-variant ImageSet", err, reqLogger) + return reconcile.Result{}, err + } else { + if nvis { + reqLogger.Info("An ImageSet exists for a different variant") + } + } + } + + if err = imageset.ValidateImageSet(imageSet); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Error validating ImageSet", err, reqLogger) + return reconcile.Result{}, err + } + + // Determine whether the Typha Deployment update must be deferred until the + // calico-node DaemonSet finishes rolling out. Per the Calico version skew + // policy, Felix may be newer than Typha, but not older: updating Typha + // while older calico-node pods are still running leaves those pods unable + // to sync, marking them NotReady, at which point the DaemonSet controller + // no longer honors the configured surge limits and replaces pods + // cluster-wide. + deferTyphaUpdate := false + desiredNodeImage, err := render.NodeImage(&instance.Spec, imageSet) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Error resolving calico-node image", err, reqLogger) + return reconcile.Result{}, err + } + nodeDS := &appsv1.DaemonSet{} + if err := r.client.Get(ctx, types.NamespacedName{Name: common.NodeDaemonSetName, Namespace: common.CalicoNamespace}, nodeDS); err != nil { + if !apierrors.IsNotFound(err) { + r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to read calico-node DaemonSet", err, reqLogger) + return reconcile.Result{}, err + } + // DaemonSet doesn't exist (fresh install) - nothing to defer for. + } else { + deferTyphaUpdate = deferTyphaDeploymentUpdate(nodeDS, desiredNodeImage) + } + if deferTyphaUpdate { + // Only defer when a Typha Deployment already exists; if it is missing + // it must be created regardless, since calico-node depends on it. + typhaDeployment := &appsv1.Deployment{} + if err := r.client.Get(ctx, types.NamespacedName{Name: common.TyphaDeploymentName, Namespace: common.CalicoNamespace}, typhaDeployment); err != nil { + if !apierrors.IsNotFound(err) { + r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to read Typha Deployment", err, reqLogger) + return reconcile.Result{}, err + } + deferTyphaUpdate = false + } + } + if deferTyphaUpdate { + reqLogger.Info("Deferring Typha Deployment update until the calico-node DaemonSet rollout completes", + "updatedNumberScheduled", nodeDS.Status.UpdatedNumberScheduled, + "desiredNumberScheduled", nodeDS.Status.DesiredNumberScheduled) + } + // Build a configuration for rendering calico/typha. typhaCfg := render.TyphaConfiguration{ K8sServiceEp: k8sapi.Endpoint, @@ -1502,6 +1569,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile ClusterDomain: r.clusterDomain, NonClusterHost: nonclusterhost, FelixHealthPort: *felixConfiguration.Spec.HealthPort, + DeferDeploymentUpdate: deferTyphaUpdate, } components = append(components, render.Typha(&typhaCfg)) @@ -1746,32 +1814,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile ) } - imageSet, err := imageset.GetImageSet(ctx, r.client, instance.Spec.Variant) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error getting ImageSet", err, reqLogger) - return reconcile.Result{}, err - } - - if imageSet == nil { - // There is no imageSet for the configured variant, but check to see if there are any - // ImageSets with a different variant so we can give the user some kind of indication - // to why an existing ImageSet is being ignored. - nvis, err := imageset.DoesNonVariantImageSetExist(ctx, r.client, instance.Spec.Variant) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error checking for non-variant ImageSet", err, reqLogger) - return reconcile.Result{}, err - } else { - if nvis { - reqLogger.Info("An ImageSet exists for a different variant") - } - } - } - - if err = imageset.ValidateImageSet(imageSet); err != nil { - r.status.SetDegraded(operatorv1.ResourceValidationError, "Error validating ImageSet", err, reqLogger) - return reconcile.Result{}, err - } - if err = imageset.ResolveImages(imageSet, components...); err != nil { r.status.SetDegraded(operatorv1.ResourceValidationError, "Error resolving ImageSet for components", err, reqLogger) return reconcile.Result{}, err @@ -1902,9 +1944,52 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } reqLogger.V(1).Info("Finished reconciling Installation") + if deferTyphaUpdate { + // Nothing reliably triggers a reconcile when the calico-node rollout + // completes: the DaemonSet watch filters out status-only updates. So + // requeue to re-evaluate the deferred Typha Deployment update. + return reconcile.Result{RequeueAfter: utils.StandardRetry}, nil + } return reconcile.Result{}, nil } +// deferTyphaDeploymentUpdate returns true when the Typha Deployment update +// must wait for the calico-node DaemonSet. While calico-node is moving to a +// different image, updating Typha would leave the not-yet-updated Felix +// instances unable to sync: per the Calico version skew policy, Felix may be +// newer than Typha, but not older. +// +// The DaemonSet is read from the informer cache before any updates are +// applied this reconcile, so the state here cannot be confused by our own +// in-flight write. The image comparison covers the reconcile that introduces +// a new image (the cached status still describes the previous rollout at that +// point); the status condition covers subsequent reconciles while the rollout +// progresses. +func deferTyphaDeploymentUpdate(nodeDS *appsv1.DaemonSet, desiredNodeImage string) bool { + var deployedImage string + for i := range nodeDS.Spec.Template.Spec.Containers { + if nodeDS.Spec.Template.Spec.Containers[i].Name == "calico-node" { + deployedImage = nodeDS.Spec.Template.Spec.Containers[i].Image + break + } + } + if deployedImage == "" { + return false + } + + // A calico-node image change is about to be applied this reconcile. + if deployedImage != desiredNodeImage { + return true + } + + // The DaemonSet spec already has the desired image; defer while its + // rollout is still in progress so that no older Felix remains when Typha + // updates. Pod readiness is deliberately not part of this condition: a + // permanently unready node must not block Typha updates indefinitely. + return nodeDS.Status.ObservedGeneration < nodeDS.Generation || + nodeDS.Status.UpdatedNumberScheduled != nodeDS.Status.DesiredNumberScheduled +} + func readMTUFile() (int, error) { filename := "/var/lib/calico/mtu" data, err := os.ReadFile(filename) diff --git a/pkg/controller/installation/typha_defer_test.go b/pkg/controller/installation/typha_defer_test.go new file mode 100644 index 0000000000..fb6e73507d --- /dev/null +++ b/pkg/controller/installation/typha_defer_test.go @@ -0,0 +1,77 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +func nodeDaemonSet(image string, generation, observedGeneration, updated, desired int64) *appsv1.DaemonSet { + ds := &appsv1.DaemonSet{} + ds.Generation = generation + ds.Spec.Template.Spec.Containers = []corev1.Container{{Name: "calico-node", Image: image}} + ds.Status.ObservedGeneration = observedGeneration + ds.Status.UpdatedNumberScheduled = int32(updated) + ds.Status.DesiredNumberScheduled = int32(desired) + return ds +} + +var _ = Describe("deferTyphaDeploymentUpdate", func() { + const ( + oldImage = "docker.io/calico/node:v3.31.3" + newImage = "docker.io/calico/node:v3.32.1" + ) + + It("does not defer in steady state (image matches, rollout complete)", func() { + ds := nodeDaemonSet(newImage, 5, 5, 30, 30) + Expect(deferTyphaDeploymentUpdate(ds, newImage)).To(BeFalse()) + }) + + It("defers when a node image change is about to be applied", func() { + // The cached DaemonSet still has the old image with a completed + // rollout; this reconcile is about to apply the new image. + ds := nodeDaemonSet(oldImage, 5, 5, 30, 30) + Expect(deferTyphaDeploymentUpdate(ds, newImage)).To(BeTrue()) + }) + + It("defers while the rollout has not been observed by the DaemonSet controller", func() { + ds := nodeDaemonSet(newImage, 6, 5, 30, 30) + Expect(deferTyphaDeploymentUpdate(ds, newImage)).To(BeTrue()) + }) + + It("defers while pods are still being updated", func() { + ds := nodeDaemonSet(newImage, 6, 6, 12, 30) + Expect(deferTyphaDeploymentUpdate(ds, newImage)).To(BeTrue()) + }) + + It("does not defer once all pods are updated, regardless of readiness", func() { + // Readiness is intentionally not part of the condition: what matters + // is that no old-version Felix remains, and a permanently unready + // node must not block Typha updates. + ds := nodeDaemonSet(newImage, 6, 6, 30, 30) + ds.Status.NumberReady = 3 + Expect(deferTyphaDeploymentUpdate(ds, newImage)).To(BeFalse()) + }) + + It("does not defer when the DaemonSet has no calico-node container", func() { + ds := nodeDaemonSet(newImage, 5, 5, 30, 30) + ds.Spec.Template.Spec.Containers = []corev1.Container{{Name: "other", Image: oldImage}} + Expect(deferTyphaDeploymentUpdate(ds, newImage)).To(BeFalse()) + }) +}) diff --git a/pkg/render/node.go b/pkg/render/node.go index e4c9857e7d..ad55ade17f 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -172,6 +172,15 @@ type nodeComponent struct { nodeImage string } +// NodeImage returns the image used by the calico-node container for the given +// installation, resolving through the ImageSet when one is present. +func NodeImage(installation *operatorv1.InstallationSpec, is *operatorv1.ImageSet) (string, error) { + if installation.Variant.IsEnterprise() { + return components.GetReference(components.ComponentTigeraNode, installation.Registry, installation.ImagePath, installation.ImagePrefix, is) + } + return components.GetReference(components.ComponentCalicoNode, installation.Registry, installation.ImagePath, installation.ImagePrefix, is) +} + func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := c.cfg.Installation.Registry path := c.cfg.Installation.ImagePath @@ -192,12 +201,7 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { c.cniPluginsImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIPlugins, reg, path, prefix, is)) } } - switch { - case c.cfg.Installation.Variant.IsEnterprise(): - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNode, reg, path, prefix, is)) - default: - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNode, reg, path, prefix, is)) - } + c.nodeImage = appendIfErr(NodeImage(c.cfg.Installation, is)) if len(errMsgs) != 0 { return fmt.Errorf("%s", strings.Join(errMsgs, ",")) diff --git a/pkg/render/typha.go b/pkg/render/typha.go index 264851785f..289ff7d0a3 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -85,6 +85,15 @@ type TyphaConfiguration struct { // The health port that Felix is bound to. We configure Typha to bind to the port // that is one less. FelixHealthPort int + + // DeferDeploymentUpdate indicates that the Typha Deployment should be left + // as-is for now: the calico-node DaemonSet is rolling out a newer version, + // and Typha must not be updated until no older Felix remains. Per the + // Calico version skew policy, Felix may be newer than Typha, but not + // older. Only the Deployment is deferred; all other Typha objects are + // still reconciled. The controller only sets this when a Typha Deployment + // already exists. + DeferDeploymentUpdate bool } // Typha creates the typha daemonset and other resources for the daemonset to operate normally. @@ -127,7 +136,11 @@ func (c *typhaComponent) Objects() ([]client.Object, []client.Object) { objs = append(objs, c.typhaServices()...) // Add deployment last, as it may depend on the creation of previous objects in the list. - objs = append(objs, c.typhaDeployment()...) + // When the update is deferred, omit the Deployment so the existing one is + // left untouched; omitted objects are not deleted by the handler. + if !c.cfg.DeferDeploymentUpdate { + objs = append(objs, c.typhaDeployment()...) + } if c.cfg.Installation.TyphaMetricsPort != nil { objs = append(objs, c.typhaPrometheusService()) }