From d58f3a79af1cda7e8a266ad9814e4eccef747e38 Mon Sep 17 00:00:00 2001 From: Alina Militaru <41362174+asincu@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:14:57 -0700 Subject: [PATCH 1/6] (bootstrap): Add knob to reconfigure Linseed indices for a single tenant cluster This will be set operator bootstrap config map and enabled when migrating from multi-index format to single-index format. Linseed will reconfigure its environment variables to set the correct backend for the indices it is using and also set the base index name used for Cloud. --- cmd/main.go | 3 +++ pkg/common/discovery/discovery.go | 16 ++++++++++++++++ .../logstorage/linseed/linseed_controller.go | 3 +++ pkg/controller/options/options.go | 4 ++++ pkg/render/logstorage/linseed/linseed.go | 8 ++++++++ 5 files changed, 34 insertions(+) diff --git a/cmd/main.go b/cmd/main.go index a1b648be36..0bf6be2e51 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -585,10 +585,12 @@ admission policy installation; once an Installation exists it is the authority o } elasticIsMigrating := false + indexIsMigrating := false useExternalElastic := discovery.UseExternalElastic(bootConfig) if isCloudBuild() { elasticIsMigrating = discovery.ElasticIsMigrating(bootConfig) + indexIsMigrating = discovery.IndexIsMigrating(bootConfig) if !elasticIsMigrating { if err := verifyElasticSearch(ctx, cs, useExternalElastic); err != nil { setupLog.Error(err, "Elasticsearch configuration verification failed") @@ -621,6 +623,7 @@ admission policy installation; once an Installation exists it is the authority o ElasticExternal: useExternalElastic, Cloud: isCloudBuild(), ESMigration: elasticIsMigrating, + IndexMigration: indexIsMigrating, UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, } diff --git a/pkg/common/discovery/discovery.go b/pkg/common/discovery/discovery.go index cc0ee2b2b5..e6b77b819b 100644 --- a/pkg/common/discovery/discovery.go +++ b/pkg/common/discovery/discovery.go @@ -306,3 +306,19 @@ func ElasticIsMigrating(config *corev1.ConfigMap) bool { } return false } + +// IndexIsMigrating returns true if this cluster is in the last phase of a migration to single-index +// storage, during which the operator must reconfigure Linseed to use the single-index names. +func IndexIsMigrating(config *corev1.ConfigMap) bool { + if config == nil { + return false + } + + // Load the operator bootstrap configuration from its configmap. + if val, ok := config.Data["INDEX_MIGRATION"]; ok && val != "" { + if strings.ToLower(val) == "true" { + return true + } + } + return false +} diff --git a/pkg/controller/logstorage/linseed/linseed_controller.go b/pkg/controller/logstorage/linseed/linseed_controller.go index 1c34680068..4792354a8c 100644 --- a/pkg/controller/logstorage/linseed/linseed_controller.go +++ b/pkg/controller/logstorage/linseed/linseed_controller.go @@ -70,6 +70,7 @@ type LinseedSubController struct { multiTenant bool elasticExternal bool cloud bool + useSingleIndex bool } func Add(mgr manager.Manager, opts options.ControllerOptions) error { @@ -89,6 +90,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { status: status.New(mgr.GetClient(), "log-storage-access", opts.KubernetesVersion), elasticExternal: opts.ElasticExternal, cloud: opts.Cloud, + useSingleIndex: opts.IndexMigration, } r.status.Run(opts.ShutdownContext) @@ -470,6 +472,7 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. ElasticClientCredentialsSecret: &credentials, LogStorage: logStorage, Cloud: r.cloud, + UseSingleIndex: r.useSingleIndex, } linseedComponent := linseed.Linseed(cfg) diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 7a7f0d8622..15bf9e7d9f 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -61,6 +61,10 @@ type ControllerOptions struct { // LSS configuration and internal elasticsearch running. Only meaningful when Cloud is set. ESMigration bool + // This field is enabled in the last phase of an index migration for a single tenant format, + // we need to reconfigure + IndexMigration bool + // Whether or not to use crd.projectcalico.org/v1 or projectcalico.org/v3 for Calico CRDs. UseV3CRDs bool diff --git a/pkg/render/logstorage/linseed/linseed.go b/pkg/render/logstorage/linseed/linseed.go index 66ea6522a9..24232aebe1 100644 --- a/pkg/render/logstorage/linseed/linseed.go +++ b/pkg/render/logstorage/linseed/linseed.go @@ -117,6 +117,7 @@ type Config struct { // Tenant configuration, if running for a particular tenant. Tenant *operatorv1.Tenant ExternalElastic bool + UseSingleIndex bool // Secret containing client certificate and key for connecting to the Elastic cluster. If configured, // mTLS is used between Linseed and the external Elastic cluster. @@ -426,6 +427,13 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { if l.cfg.Tenant.Spec.ControlPlaneReplicas != nil { replicas = l.cfg.Tenant.Spec.ControlPlaneReplicas } + } else if l.cfg.UseSingleIndex { + // For single-tenant clusters migrating to multi-tenant style indices, + // use the elastic-single-tenant backend and configure index base names. + envVars = append(envVars, corev1.EnvVar{Name: "BACKEND", Value: "elastic-single-tenant"}) + for _, index := range l.cfg.Tenant.Spec.Indices { + envVars = append(envVars, index.EnvVar()) + } } } sc := securitycontext.NewNonRootContext() From ff5694dab612ab2b310f11e3203fe7a900f8be64 Mon Sep 17 00:00:00 2001 From: Alina Militaru <41362174+asincu@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:06:28 -0700 Subject: [PATCH 2/6] (logstorage-users): Provision Elasticsearch users from the operator while migrating to single-index storage Single-tenant clusters migrating to single-index storage need Linseed to hold RBAC for the new calico_* indices. es-kube-controllers cannot grant that, so during the migration the operator's log-storage users controller takes over user provisioning and es-kube-controllers stops running its elasticsearch configuration controller. - Run the users controller in single-tenant mode when IndexMigration is set, building the tenant configuration from the cloud config ConfigMap since single-tenant clusters have no Tenant resource. - Name the single-tenant Linseed and Dashboards users the way es-kube-controllers named them (--secure), and repoint existing credential secrets at those users while keeping their passwords, so credentials provisioned before the migration keep resolving. - Declare the standard single-index names on the Tenant that CloudConfig.ToTenant builds, gated on the caller opting in, so that clusters which are not migrating keep falling back to their existing index names. Sort the declared indices, as they are generated from a map. - Report the users TigeraStatus in the log-storage conditions aggregate while migrating. --- api/v1/tenant_types.go | 17 +++ .../intrusiondetection_controller.go | 2 +- .../dashboards/dashboards_controller.go | 5 +- .../initializer/conditions_controller.go | 15 +- .../logstorage/kubecontrollers/cloud.go | 5 + .../kubecontrollers/es_kube_controllers.go | 2 + .../logstorage/linseed/linseed_controller.go | 8 +- .../logstorage/users/users_controller.go | 120 ++++++++++++---- .../logstorage/users/users_controller_test.go | 129 ++++++++++++++++++ .../manager/manager_controller_cloud.go | 2 +- .../policyrecommendation_controller.go | 2 +- pkg/controller/utils/elasticsearch.go | 35 ++++- pkg/render/common/cloudconfig/cloudconfig.go | 25 +++- .../common/cloudconfig/cloudconfig_test.go | 47 +++++++ .../kubecontrollers/kube-controllers.go | 12 +- .../kubecontrollers/kube-controllers_test.go | 18 +++ pkg/render/logstorage/linseed/linseed.go | 4 +- 17 files changed, 401 insertions(+), 47 deletions(-) diff --git a/api/v1/tenant_types.go b/api/v1/tenant_types.go index 89f2297df6..2ea355ef09 100644 --- a/api/v1/tenant_types.go +++ b/api/v1/tenant_types.go @@ -67,6 +67,23 @@ var DataTypes = map[DataType]string{ DataTypePolicyActivity: "ELASTIC_POLICY_ACTIVITY_BASE_INDEX_NAME", } +var CloudStandardIndices = map[DataType]string{ + DataTypeAlerts: "calico_alerts_standard", + DataTypeAuditLogs: "calico_auditlogs_standard", + DataTypeBGPLogs: "calico_bgplogs_standard", + DataTypeComplianceBenchmarks: "calico_compliance_benchmarks_results_standard", + DataTypeComplianceReports: "calico_compliance_reports_standard", + DataTypeComplianceSnapshots: "calico_compliance_snapshots_standard", + DataTypeDNSLogs: "calico_dnslogs_standard", + DataTypeFlowLogs: "calico_flowlogs_standard", + DataTypeL7Logs: "calico_l7logs_standard", + DataTypeRuntimeReports: "calico_runtime_reports_standard", + DataTypeThreatFeedsDomainSet: "calico_threatfeeds_domainnameset_standard", + DataTypeThreatFeedsIPSet: "calico_threatfeeds_ipset_standard", + DataTypeWAFLogs: "calico_waflogs_standard", + DataTypePolicyActivity: "calico_policy_activity_standard", +} + type TenantSpec struct { // ID is the unique identifier for this tenant. // +required diff --git a/pkg/controller/intrusiondetection/intrusiondetection_controller.go b/pkg/controller/intrusiondetection/intrusiondetection_controller.go index f46915a891..68ce7b538f 100644 --- a/pkg/controller/intrusiondetection/intrusiondetection_controller.go +++ b/pkg/controller/intrusiondetection/intrusiondetection_controller.go @@ -437,7 +437,7 @@ func (r *ReconcileIntrusionDetection) Reconcile(ctx context.Context, request rec r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } // Create a component handler to manage the rendered component. diff --git a/pkg/controller/logstorage/dashboards/dashboards_controller.go b/pkg/controller/logstorage/dashboards/dashboards_controller.go index 9bf035c1d7..eecab837a8 100644 --- a/pkg/controller/logstorage/dashboards/dashboards_controller.go +++ b/pkg/controller/logstorage/dashboards/dashboards_controller.go @@ -290,7 +290,7 @@ func (d DashboardsSubController) Reconcile(ctx context.Context, request reconcil d.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } // Determine the host and port from the URL. @@ -323,7 +323,8 @@ func (d DashboardsSubController) Reconcile(ctx context.Context, request reconcil // Query the username and password this Dashboards Installer instance should use to authenticate with Elasticsearch. // For multi-tenant systems, credentials are created by the elasticsearch users controller. - // For single-tenant system, these are created by es-kube-controllers. + // For single-tenant systems, these are created by es-kube-controllers, unless the cluster is + // migrating to single-index storage - then the users controller creates them as well. key = types.NamespacedName{Name: dashboards.ElasticCredentialsSecret, Namespace: helper.InstallNamespace()} credentials := corev1.Secret{} if err = d.client.Get(ctx, key, &credentials); err != nil && !errors.IsNotFound(err) { diff --git a/pkg/controller/logstorage/initializer/conditions_controller.go b/pkg/controller/logstorage/initializer/conditions_controller.go index 847a589b68..fd66412174 100644 --- a/pkg/controller/logstorage/initializer/conditions_controller.go +++ b/pkg/controller/logstorage/initializer/conditions_controller.go @@ -41,9 +41,10 @@ func AddConditionsController(mgr manager.Manager, opts options.ControllerOptions // Create the reconciler r := &LogStorageConditions{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - multiTenant: opts.MultiTenant, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + multiTenant: opts.MultiTenant, + indexMigration: opts.IndexMigration, } return ctrl.NewControllerManagedBy(mgr). @@ -62,6 +63,10 @@ type LogStorageConditions struct { client client.Client scheme *runtime.Scheme multiTenant bool + + // indexMigration indicates that this single-tenant cluster is migrating to single-index storage, + // in which case the log-storage users controller runs and reports status. + indexMigration bool } func (r *LogStorageConditions) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { @@ -112,6 +117,10 @@ func (r *LogStorageConditions) getDesiredConditions(ctx context.Context) (map[st expectedInstances = append(expectedInstances, TigeraStatusLogStorageUsers) } else { expectedInstances = append(expectedInstances, TigeraStatusLogStorageESMetrics, TigeraStatusLogStorageKubeController, TigeraStatusLogStorageDashboards) + if r.indexMigration { + // While migrating to single-index storage, the users controller runs in single-tenant mode too. + expectedInstances = append(expectedInstances, TigeraStatusLogStorageUsers) + } } // Keep track of which instances are in which state. diff --git a/pkg/controller/logstorage/kubecontrollers/cloud.go b/pkg/controller/logstorage/kubecontrollers/cloud.go index 93a4c4110d..19a183f95b 100644 --- a/pkg/controller/logstorage/kubecontrollers/cloud.go +++ b/pkg/controller/logstorage/kubecontrollers/cloud.go @@ -70,6 +70,11 @@ func (r *ESKubeControllersController) esGatewayAddCloudModificationsToConfig(c * // esKubeControllersAddCloudModificationsToConfig modifies the provided *kubecontrollers.KubeControllersConfiguration to include Calico Cloud specific configuration. func (r *ESKubeControllersController) esKubeControllersAddCloudModificationsToConfig(c *kubecontrollers.KubeControllersConfiguration, reqLogger logr.Logger, ctx context.Context) (reconcile.Result, bool, error) { + // While migrating to single-index storage, es-kube-controllers must not run the elasticsearch + // configuration controller. The operator's log-storage users controller provisions the + // Elasticsearch users instead, so that Linseed gets the RBAC needed for the new indices. + c.IndexMigration = r.indexMigration + if r.cloud && r.elasticExternal && !r.multiTenant { cloudConfig, err := utils.GetCloudConfig(ctx, r.client) if err != nil { diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index 735685963c..cd1e227306 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -64,6 +64,7 @@ type ESKubeControllersController struct { elasticExternal bool multiTenant bool cloud bool + indexMigration bool tierWatchReady *utils.ReadyFlag } @@ -90,6 +91,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { elasticExternal: opts.ElasticExternal, multiTenant: opts.MultiTenant, cloud: opts.Cloud, + indexMigration: opts.IndexMigration, tierWatchReady: &utils.ReadyFlag{}, } r.status.Run(opts.ShutdownContext) diff --git a/pkg/controller/logstorage/linseed/linseed_controller.go b/pkg/controller/logstorage/linseed/linseed_controller.go index 4792354a8c..d780b090ae 100644 --- a/pkg/controller/logstorage/linseed/linseed_controller.go +++ b/pkg/controller/logstorage/linseed/linseed_controller.go @@ -338,7 +338,7 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(r.useSingleIndex) } // Determine the host and port from the URL. @@ -365,7 +365,8 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. // Query the username and password this Linseed instance should use to authenticate with Elasticsearch. // For multi-tenant systems, credentials are created by the elasticsearch users controller. - // For single-tenant system, these are created by es-kube-controllers. + // For single-tenant systems, these are created by es-kube-controllers, unless the cluster is + // migrating to single-index storage - then the users controller creates them as well. key = types.NamespacedName{Name: render.ElasticsearchLinseedUserSecret, Namespace: helper.InstallNamespace()} credentials := corev1.Secret{} if err = r.client.Get(ctx, key, &credentials); err != nil && !errors.IsNotFound(err) { @@ -422,7 +423,8 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. // Query the username and password this Linseed instance should use to authenticate with Elasticsearch. // For multi-tenant systems, credentials are created by the elasticsearch users controller. - // For single-tenant system, these are created by es-kube-controllers. + // For single-tenant systems, these are created by es-kube-controllers, unless the cluster is + // migrating to single-index storage - then the users controller creates them as well. // Delay installing Linseed until available. // TODO: Switch single-tenant to using operator-provisioned users. key = types.NamespacedName{Name: render.ElasticsearchLinseedUserSecret, Namespace: helper.InstallNamespace()} diff --git a/pkg/controller/logstorage/users/users_controller.go b/pkg/controller/logstorage/users/users_controller.go index c1deaf9e1f..3018c341e9 100644 --- a/pkg/controller/logstorage/users/users_controller.go +++ b/pkg/controller/logstorage/users/users_controller.go @@ -28,11 +28,13 @@ import ( "github.com/tigera/operator/pkg/render/logstorage/dashboards" corev1 "k8s.io/api/core/v1" + "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/crypto" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/common/secret" "k8s.io/apimachinery/pkg/api/errors" @@ -59,6 +61,11 @@ type UserController struct { esClientFn utils.ElasticsearchClientCreator multiTenant bool elasticExternal bool + + // indexMigration indicates that this single-tenant cluster is migrating to single-index storage. + // While migrating, this controller provisions the Elasticsearch users instead of es-kube-controllers, + // so that Linseed gets the RBAC needed for the new indices. + indexMigration bool } type UsersCleanupController struct { @@ -72,9 +79,10 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if !opts.Variant.IsEnterprise() { return nil } - if !opts.MultiTenant { - // For now, the operator only creates users in multi-tenant mode. In single-tenant mode, - // user creation is handled by es-kube-controllers instead. + if !opts.MultiTenant && !opts.IndexMigration { + // The operator only creates users in multi-tenant mode, or in single-tenant mode while + // migrating to single-index storage. Otherwise, user creation is handled by + // es-kube-controllers instead. return nil } @@ -83,6 +91,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { client: mgr.GetClient(), scheme: mgr.GetScheme(), multiTenant: opts.MultiTenant, + indexMigration: opts.IndexMigration, status: status.New(mgr.GetClient(), initializer.TigeraStatusLogStorageUsers, opts.KubernetesVersion), esClientFn: utils.NewElasticClient, elasticExternal: opts.ElasticExternal, @@ -119,6 +128,12 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err = c.WatchObject(&operatorv1.Tenant{}, &handler.EnqueueRequestForObject{}); err != nil { return fmt.Errorf("log-storage-user-controller failed to watch Tenant resource: %w", err) } + } else { + // In single-tenant mode, the tenant configuration - including the Elasticsearch endpoint - + // comes from the cloud config ConfigMap. + if err = utils.AddConfigMapWatch(c, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("log-storage-user-controller failed to watch the ConfigMap resource: %w", err) + } } // Watch for Elasticsearch. @@ -133,6 +148,11 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("log-storage-user-controller failed to create periodic reconcile watch: %w", err) } + if !opts.MultiTenant { + // The cleanup controller reconciles Tenant resources, which only exist in multi-tenant mode. + return nil + } + // Now that the users controller is set up, we can also set up the controller that cleans up stale users usersCleanupReconciler := &UsersCleanupController{ client: mgr.GetClient(), @@ -179,6 +199,18 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques return reconcile.Result{}, err } + if !r.multiTenant { + // Single-tenant clusters have no Tenant resource. Build the equivalent tenant configuration + // from the cloud config, so that we provision the users against the right Elasticsearch. + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) + return reconcile.Result{}, err + } + tenant = cloudConfig.ToTenant(r.indexMigration) + tenantID = tenant.Spec.ID + } + // Get LogStorage resource. logStorage := &operatorv1.LogStorage{} err = r.client.Get(ctx, utils.DefaultEnterpriseInstanceKey, logStorage) @@ -215,33 +247,25 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques } } - clusterIDConfigMap := corev1.ConfigMap{} - clusterIDConfigMapKey := client.ObjectKey{Name: "cluster-info", Namespace: "tigera-operator"} - err = r.client.Get(ctx, clusterIDConfigMapKey, &clusterIDConfigMap) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Waiting for ConfigMap %s/%s to be available", clusterIDConfigMapKey.Namespace, clusterIDConfigMapKey.Name), - nil, reqLogger) - return reconcile.Result{}, err - } - - clusterID, ok := clusterIDConfigMap.Data["cluster-id"] - if !ok { - err = fmt.Errorf("%s/%s ConfigMap does not contain expected 'cluster-id' key", - clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) - return reconcile.Result{}, err - } - - if clusterID == "" { - err = fmt.Errorf("%s/%s ConfigMap value for key 'cluster-id' must be non-empty", - clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) - return reconcile.Result{}, err + // Determine the names of the users to provision. In multi-tenant clusters the cluster ID forms part + // of the user names, and is read from the cluster-info ConfigMap written at install time. + // Single-tenant clusters have no such ConfigMap - their user names are derived from the tenant ID + // alone, matching the names es-kube-controllers used before the operator took over provisioning. + var linseedUser, dashboardUser *utils.User + if r.multiTenant { + clusterID, err := r.clusterID(ctx, reqLogger) + if err != nil { + return reconcile.Result{}, err + } + linseedUser = utils.LinseedUser(clusterID, tenantID) + dashboardUser = utils.DashboardUser(clusterID, tenantID) + } else { + linseedUser = utils.LinseedUserSingleTenant(tenantID) + dashboardUser = utils.DashboardUserSingleTenant(tenantID) } // Query any existing username and password for this Linseed instance. If one already exists, we'll simply // use that. Otherwise, generate a new one. - linseedUser := utils.LinseedUser(clusterID, tenantID) linseedUserSecret := corev1.Secret{} var credentialSecrets []client.Object key := types.NamespacedName{Name: render.ElasticsearchLinseedUserSecret, Namespace: helper.TruthNamespace()} @@ -256,14 +280,19 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Make sure we install the generated credentials into the truth namespace. credentialSecrets = append(credentialSecrets, &linseedUserSecret) + } else if string(linseedUserSecret.Data["username"]) != linseedUser.Username { + // The credentials exist, but reference a different Elasticsearch user than the one we provision - + // e.g. because they were created by es-kube-controllers before this cluster started migrating to + // single-index storage. Point them at our user, keeping the existing password. + linseedUserSecret.StringData = map[string]string{"username": linseedUser.Username} + credentialSecrets = append(credentialSecrets, &linseedUserSecret) } // Query any existing username and password for this Dashboards instance. If one already exists, we'll simply // use that. Otherwise, generate a new one. keyDashboardCred := types.NamespacedName{Name: dashboards.ElasticCredentialsSecret, Namespace: helper.TruthNamespace()} - dashboardUser := utils.DashboardUser(clusterID, tenantID) dashboardUserSecret := corev1.Secret{} - if err = r.client.Get(ctx, key, &dashboardUserSecret); err != nil && !errors.IsNotFound(err) { + if err = r.client.Get(ctx, keyDashboardCred, &dashboardUserSecret); err != nil && !errors.IsNotFound(err) { r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error getting Secret %s", keyDashboardCred), err, reqLogger) return reconcile.Result{}, err } else if errors.IsNotFound(err) { @@ -274,6 +303,10 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Make sure we install the generated credentials into the truth namespace. credentialSecrets = append(credentialSecrets, &dashboardUserSecret) + } else if string(dashboardUserSecret.Data["username"]) != dashboardUser.Username { + // As above - point the existing credentials at the user we provision. + dashboardUserSecret.StringData = map[string]string{"username": dashboardUser.Username} + credentialSecrets = append(credentialSecrets, &dashboardUserSecret) } if helper.TruthNamespace() != helper.InstallNamespace() { @@ -297,7 +330,7 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Add a finalizer to the Tenant instance if it exists so that we can clean up the Linseed user when the Tenant // is deleted. The finalizer will be removed by the user cleanup controller when the user is deleted from ES. - if tenant != nil && tenant.GetDeletionTimestamp().IsZero() && !stringsutil.StringInSlice(userCleanupFinalizer, tenant.GetFinalizers()) { + if r.multiTenant && tenant != nil && tenant.GetDeletionTimestamp().IsZero() && !stringsutil.StringInSlice(userCleanupFinalizer, tenant.GetFinalizers()) { tenant.SetFinalizers(append(tenant.GetFinalizers(), userCleanupFinalizer)) if err = r.client.Update(ctx, tenant); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error adding finalizer to Tenant", err, reqLogger) @@ -325,6 +358,35 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques return reconcile.Result{}, nil } +// clusterID reads the cluster ID from the cluster-info ConfigMap. This ConfigMap is written at install +// time and only exists in multi-tenant clusters. +func (r *UserController) clusterID(ctx context.Context, reqLogger logr.Logger) (string, error) { + clusterIDConfigMap := corev1.ConfigMap{} + clusterIDConfigMapKey := client.ObjectKey{Name: "cluster-info", Namespace: "tigera-operator"} + if err := r.client.Get(ctx, clusterIDConfigMapKey, &clusterIDConfigMap); err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Waiting for ConfigMap %s/%s to be available", clusterIDConfigMapKey.Namespace, clusterIDConfigMapKey.Name), + nil, reqLogger) + return "", err + } + + clusterID, ok := clusterIDConfigMap.Data["cluster-id"] + if !ok { + err := fmt.Errorf("%s/%s ConfigMap does not contain expected 'cluster-id' key", + clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) + r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) + return "", err + } + + if clusterID == "" { + err := fmt.Errorf("%s/%s ConfigMap value for key 'cluster-id' must be non-empty", + clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) + r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) + return "", err + } + + return clusterID, nil +} + func (r *UserController) createUserLogin(ctx context.Context, elasticEndpoint string, secret *corev1.Secret, user *utils.User, reqLogger logr.Logger) error { esClient, err := r.esClientFn(r.client, ctx, elasticEndpoint, r.elasticExternal) if err != nil { diff --git a/pkg/controller/logstorage/users/users_controller_test.go b/pkg/controller/logstorage/users/users_controller_test.go index 1a8620c2f7..70605f8228 100644 --- a/pkg/controller/logstorage/users/users_controller_test.go +++ b/pkg/controller/logstorage/users/users_controller_test.go @@ -23,15 +23,23 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" apiv1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" tigeraelastic "github.com/tigera/operator/pkg/controller/logstorage/elastic" + "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" + "github.com/tigera/operator/pkg/render/logstorage/dashboards" ) var _ = Describe("LogStorage cleanup controller", func() { @@ -110,3 +118,124 @@ var _ = Describe("LogStorage cleanup controller", func() { Expect(testESClient.AssertExpectations(t)) }) }) + +// fakeESClient records the users provisioned against Elasticsearch. +type fakeESClient struct { + created []*utils.User +} + +func (f *fakeESClient) SetILMPolicies(context.Context, *operatorv1.LogStorage, bool) error { + return nil +} + +func (f *fakeESClient) CreateUser(_ context.Context, user *utils.User) error { + f.created = append(f.created, user) + return nil +} + +func (f *fakeESClient) DeleteUser(context.Context, *utils.User) error { return nil } + +func (f *fakeESClient) GetUsers(context.Context) ([]utils.User, error) { return nil, nil } + +var _ = Describe("LogStorage users controller", func() { + const ( + tenantID = "tenant-a" + ) + + var ( + cli client.Client + ctx context.Context + scheme *runtime.Scheme + esClient *fakeESClient + r *UserController + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(operatorv1.AddToScheme(scheme)).NotTo(HaveOccurred()) + Expect(corev1.AddToScheme(scheme)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + ctx = context.Background() + + ls := &operatorv1.LogStorage{} + ls.Name = "tigera-secure" + ls.Status.State = operatorv1.TigeraStatusReady + Expect(cli.Create(ctx, ls)).NotTo(HaveOccurred()) + + // Note that no cluster-info ConfigMap is created here: single-tenant clusters don't have one, + // and the controller must not require it. + Expect(cli.Create(ctx, cloudconfig.NewCloudConfig(tenantID, "tenant-a-name", "es.example.com", "kb.example.com", false).ConfigMap())).NotTo(HaveOccurred()) + + mockStatus := &status.MockStatus{} + mockStatus.On("OnCRFound").Return() + mockStatus.On("ReadyToMonitor") + mockStatus.On("ClearDegraded") + mockStatus.On("ClearWarning", mock.Anything).Return() + mockStatus.On("SetDegraded", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + + esClient = &fakeESClient{} + r = &UserController{ + client: cli, + scheme: scheme, + status: mockStatus, + esClientFn: func(_ client.Client, _ context.Context, _ string, _ bool) (utils.ElasticClient, error) { + return esClient, nil + }, + multiTenant: false, + elasticExternal: true, + indexMigration: true, + } + }) + + // secretValue reads a value from a Secret. The fake client doesn't convert StringData into Data + // the way the API server does, so we may find the value in either field. + secretValue := func(name, namespace, key string) string { + s := &corev1.Secret{} + ExpectWithOffset(1, cli.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, s)).NotTo(HaveOccurred()) + if v, ok := s.StringData[key]; ok { + return v + } + return string(s.Data[key]) + } + + It("should provision users for a single-tenant cluster migrating to single-index storage", func() { + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + + // The Linseed user should be created in ES with the name es-kube-controllers used, and with + // privileges on the new single-index calico_* indices. + expected := utils.LinseedUserSingleTenant(tenantID) + Expect(expected.Username).To(Equal("tigera-ee-linseed-tenant-a-secure")) + Expect(esClient.created).To(HaveLen(2)) + Expect(esClient.created[0].Username).To(Equal(expected.Username)) + Expect(esClient.created[0].Roles).To(Equal(expected.Roles)) + Expect(esClient.created[0].Roles[0].Name).To(Equal(expected.Username)) + Expect(esClient.created[0].Roles[0].Definition.Indices[0].Names).To(ContainElement("calico_*")) + Expect(esClient.created[1].Username).To(Equal("tigera-ee-dashboards-installer-tenant-a-secure")) + + // The credentials should be written to the Elasticsearch namespace for Linseed to consume. + Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "username")).To(Equal(expected.Username)) + Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "password")).NotTo(BeEmpty()) + Expect(secretValue(dashboards.ElasticCredentialsSecret, render.ElasticsearchNamespace, "username")).To(Equal(utils.DashboardUserSingleTenant(tenantID).Username)) + }) + + It("should re-point existing credentials at the operator provisioned user", func() { + // es-kube-controllers uses a different naming convention for the ES user. + Expect(cli.Create(ctx, &corev1.Secret{ + ObjectMeta: apiv1.ObjectMeta{Name: render.ElasticsearchLinseedUserSecret, Namespace: common.OperatorNamespace()}, + Data: map[string][]byte{"username": []byte("tigera-ee-linseed"), "password": []byte("existing-password")}, + })).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + + expected := utils.LinseedUserSingleTenant(tenantID) + Expect(secretValue(render.ElasticsearchLinseedUserSecret, common.OperatorNamespace(), "username")).To(Equal(expected.Username)) + Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "username")).To(Equal(expected.Username)) + + // The existing password is preserved, and used for the user we provision. + Expect(secretValue(render.ElasticsearchLinseedUserSecret, common.OperatorNamespace(), "password")).To(Equal("existing-password")) + Expect(esClient.created[0].Username).To(Equal(expected.Username)) + Expect(esClient.created[0].Password).To(Equal("existing-password")) + }) +}) diff --git a/pkg/controller/manager/manager_controller_cloud.go b/pkg/controller/manager/manager_controller_cloud.go index 5f0c56591b..e9384b96b6 100644 --- a/pkg/controller/manager/manager_controller_cloud.go +++ b/pkg/controller/manager/manager_controller_cloud.go @@ -122,7 +122,7 @@ func (r *ReconcileManager) handleCloudReconcile( r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return nil, render.ManagerCloudResources{}, nil, nil, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } else { var err error tenant, err = utils.GetTenantFromCloudAuthConfig(ctx, r.client) diff --git a/pkg/controller/policyrecommendation/policyrecommendation_controller.go b/pkg/controller/policyrecommendation/policyrecommendation_controller.go index 3fdd36a259..b10c84d3f7 100644 --- a/pkg/controller/policyrecommendation/policyrecommendation_controller.go +++ b/pkg/controller/policyrecommendation/policyrecommendation_controller.go @@ -349,7 +349,7 @@ func (r *ReconcilePolicyRecommendation) Reconcile(ctx context.Context, request r r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, logc) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } // Create a component handler to manage the rendered component. diff --git a/pkg/controller/utils/elasticsearch.go b/pkg/controller/utils/elasticsearch.go index 0a977f291b..bbe4cde489 100644 --- a/pkg/controller/utils/elasticsearch.go +++ b/pkg/controller/utils/elasticsearch.go @@ -220,8 +220,30 @@ var ( ElasticsearchUserNameDashboardInstaller = "tigera-ee-dashboards-installer" ) +// ElasticsearchSecureUserSuffix is appended to the user names provisioned for single-tenant clusters. +// It maintains the 1:1 mapping between the public user propagated to components and the private user +// swapped in at ES gateway, which strips this suffix. +const ElasticsearchSecureUserSuffix = "secure" + +// formatNameSingleTenant builds the ES user name for a single-tenant cluster: --secure. +// This matches the name previously provisioned by es-kube-controllers, so that existing credentials +// and role mappings keep resolving once the operator takes over user provisioning. +func formatNameSingleTenant(name, tenantID string) string { + return fmt.Sprintf("%s-%s-%s", name, tenantID, ElasticsearchSecureUserSuffix) +} + func LinseedUser(clusterID, tenant string) *User { - username := formatName(ElasticsearchUserNameLinseed, clusterID, tenant) + return linseedUser(formatName(ElasticsearchUserNameLinseed, clusterID, tenant), tenant) +} + +// LinseedUserSingleTenant returns the Linseed user for a single-tenant cluster. It carries the same +// privileges as the multi-tenant user - including access to the single-index calico_* indices - but is +// named the way es-kube-controllers named it, and needs no cluster ID. +func LinseedUserSingleTenant(tenant string) *User { + return linseedUser(formatNameSingleTenant(ElasticsearchUserNameLinseed, tenant), tenant) +} + +func linseedUser(username, tenant string) *User { return &User{ Username: username, Roles: []Role{ @@ -243,7 +265,16 @@ func LinseedUser(clusterID, tenant string) *User { } func DashboardUser(clusterID, tenant string) *User { - username := formatName(ElasticsearchUserNameDashboardInstaller, clusterID, tenant) + return dashboardUser(formatName(ElasticsearchUserNameDashboardInstaller, clusterID, tenant)) +} + +// DashboardUserSingleTenant returns the Dashboards installer user for a single-tenant cluster, named +// the way es-kube-controllers named it. See LinseedUserSingleTenant. +func DashboardUserSingleTenant(tenant string) *User { + return dashboardUser(formatNameSingleTenant(ElasticsearchUserNameDashboardInstaller, tenant)) +} + +func dashboardUser(username string) *User { return &User{ Username: username, Roles: []Role{ diff --git a/pkg/render/common/cloudconfig/cloudconfig.go b/pkg/render/common/cloudconfig/cloudconfig.go index f93dd3209e..7a5a530529 100644 --- a/pkg/render/common/cloudconfig/cloudconfig.go +++ b/pkg/render/common/cloudconfig/cloudconfig.go @@ -16,6 +16,7 @@ package cloudconfig import ( "fmt" + "sort" "strconv" v1 "github.com/tigera/operator/api/v1" @@ -82,8 +83,12 @@ type CloudConfig struct { // ToTenant converts the given CloudConfig structure to a Tenant object. // This allows controllers that have been converted to support multi-tenancy to still leverage // the single-tenant CloudConfig structure using the same code path as in multi-tenancy. -func (c CloudConfig) ToTenant() *v1.Tenant { - return &v1.Tenant{ +// +// useSingleIndex declares the standard single-index base names on the returned Tenant. Only clusters +// migrating to single-index storage should set it: index base names are otherwise not carried on the +// artificial single-tenant Tenant, and components fall back to their default index names. +func (c CloudConfig) ToTenant(useSingleIndex bool) *v1.Tenant { + tenant := &v1.Tenant{ // We don't specify a Namespace for this tenant because it represents a singular tenant installed // in this management cluster. The signals to the render code that this is a single-tenant cluster and not // a cluster capable of multi-tenancy. @@ -98,6 +103,22 @@ func (c CloudConfig) ToTenant() *v1.Tenant { }, }, } + + if !useSingleIndex { + return tenant + } + + for dataType := range v1.DataTypes { + tenant.Spec.Indices = append(tenant.Spec.Indices, v1.Index{DataType: dataType, BaseIndexName: v1.CloudStandardIndices[dataType]}) + } + + // DataTypes is a map, so iteration order is random. Sort by data type to keep the generated + // index list - and therefore the env vars rendered from it - stable across reconciles. + sort.Slice(tenant.Spec.Indices, func(i, j int) bool { + return tenant.Spec.Indices[i].DataType < tenant.Spec.Indices[j].DataType + }) + + return tenant } func (c CloudConfig) TenantId() string { diff --git a/pkg/render/common/cloudconfig/cloudconfig_test.go b/pkg/render/common/cloudconfig/cloudconfig_test.go index da125c5d73..f8d8d5f23d 100644 --- a/pkg/render/common/cloudconfig/cloudconfig_test.go +++ b/pkg/render/common/cloudconfig/cloudconfig_test.go @@ -17,6 +17,7 @@ package cloudconfig import ( "strconv" + v1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" corev1 "k8s.io/api/core/v1" @@ -91,6 +92,52 @@ var _ = Describe("CloudConfig ConfigMap tests", func() { }) }) + Context("ToTenant", func() { + var cloudConfig *CloudConfig + + BeforeEach(func() { + cloudConfig = &CloudConfig{ + tenantId: "abc123", + tenantName: "tenant1", + externalESDomain: "externalES.com", + externalKibanaDomain: "externalKibana.com", + enableMTLS: true, + } + }) + + It("should return a single-tenant Tenant with the Elastic configuration from the CloudConfig", func() { + tenant := cloudConfig.ToTenant(false) + Expect(tenant.Name).To(Equal("default")) + Expect(tenant.Namespace).To(BeEmpty()) + Expect(tenant.MultiTenant()).To(BeFalse()) + Expect(tenant.Spec.ID).To(Equal("abc123")) + Expect(tenant.Spec.Name).To(Equal("tenant1")) + Expect(tenant.Spec.Elastic.URL).To(Equal("https://externalES.com:443")) + Expect(tenant.Spec.Elastic.KibanaURL).To(Equal("https://externalKibana.com:443")) + Expect(tenant.Spec.Elastic.MutualTLS).To(BeTrue()) + }) + + It("should not declare any indices when not using single-index storage", func() { + Expect(cloudConfig.ToTenant(false).Spec.Indices).To(BeEmpty()) + }) + + It("should declare the standard index for every data type when using single-index storage", func() { + indices := cloudConfig.ToTenant(true).Spec.Indices + Expect(indices).To(HaveLen(len(v1.DataTypes))) + for _, index := range indices { + Expect(index.BaseIndexName).To(Equal(v1.CloudStandardIndices[index.DataType])) + Expect(index.BaseIndexName).ToNot(BeEmpty()) + } + }) + + It("should declare indices in a stable order", func() { + expected := cloudConfig.ToTenant(true).Spec.Indices + for i := 0; i < 10; i++ { + Expect(cloudConfig.ToTenant(true).Spec.Indices).To(Equal(expected)) + } + }) + }) + Context("ConfigMap from CloudConfig", func() { var cloudConfig *CloudConfig diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index f1c5e1f14e..ea9e1d6dd8 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -125,6 +125,11 @@ type KubeControllersConfiguration struct { // cloud-specific RBAC below is not granted and enterprise RBAC is unchanged. Cloud bool + // IndexMigration indicates that this cluster is migrating to single-index storage. While migrating, + // Elasticsearch configuration - including provisioning of the Linseed user - is handled by the + // operator's log-storage users controller rather than by es-kube-controllers. + IndexMigration bool + MetricsServerTLS certificatemanagement.KeyPairInterface // Namespace to be installed into. @@ -282,7 +287,12 @@ func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeCon var enabledControllers []string if !cfg.Tenant.MultiTenant() { // Zero and single tenant cluster needs elasticsearch configuration - enabledControllers = append(enabledControllers, "authorization", "elasticsearchconfiguration") + enabledControllers = append(enabledControllers, "authorization") + if !cfg.IndexMigration { + // While migrating to single-index storage, the operator's log-storage users controller + // provisions the Elasticsearch users so that they get the RBAC needed for the new indices. + enabledControllers = append(enabledControllers, "elasticsearchconfiguration") + } if cfg.ManagementCluster != nil && cfg.Tenant == nil { // Enterprise will require the managedcluster controller to push licenses enabledControllers = append(enabledControllers, "managedcluster") diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 80db91bff7..f103f28ba9 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -567,6 +567,24 @@ var _ = Describe("kube-controllers rendering tests", func() { })) }) + It("should not enable the elasticsearchconfiguration controller when migrating to single-index storage", func() { + instance.Variant = operatorv1.TigeraSecureEnterprise + cfg.LogStorageExists = true + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + cfg.IndexMigration = true + + component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + envs := dp.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "authorization", + })) + }) + It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { expectedResources := []struct { name string diff --git a/pkg/render/logstorage/linseed/linseed.go b/pkg/render/logstorage/linseed/linseed.go index 24232aebe1..66949407a9 100644 --- a/pkg/render/logstorage/linseed/linseed.go +++ b/pkg/render/logstorage/linseed/linseed.go @@ -429,8 +429,8 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { } } else if l.cfg.UseSingleIndex { // For single-tenant clusters migrating to multi-tenant style indices, - // use the elastic-single-tenant backend and configure index base names. - envVars = append(envVars, corev1.EnvVar{Name: "BACKEND", Value: "elastic-single-tenant"}) + // use the elastic-single-index backend and configure index base names. + envVars = append(envVars, corev1.EnvVar{Name: "BACKEND", Value: "elastic-single-index"}) for _, index := range l.cfg.Tenant.Spec.Indices { envVars = append(envVars, index.EnvVar()) } From b78f680e1bbacd90cb7c794b40d5657a06fd46cd Mon Sep 17 00:00:00 2001 From: Alina Militaru <41362174+asincu@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:09:06 -0700 Subject: [PATCH 3/6] (logstorage-conditions): Stop the log-storage conditions controller from writing on every reconcile updateConditions built its result by ranging over the desiredConditions map, so the order of LogStorage.Status.Conditions was randomized on every reconcile. Conditions is an atomic list, so a reorder is a real change to the stored object: each reconcile bumped the resourceVersion, and since this controller also watches LogStorage, that re-enqueued itself. The write loop ran continuously, and reconciles fired faster than the informer cache could converge - so reconciles read a stale tigera-secure and their status updates were rejected with "the object has been modified". Sort the conditions by type so the stored list is stable, skip the status update entirely when the computed conditions match what is already stored, and requeue instead of erroring when an update does hit a conflict. --- .../initializer/conditions_controller.go | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/controller/logstorage/initializer/conditions_controller.go b/pkg/controller/logstorage/initializer/conditions_controller.go index fd66412174..e35b953acd 100644 --- a/pkg/controller/logstorage/initializer/conditions_controller.go +++ b/pkg/controller/logstorage/initializer/conditions_controller.go @@ -17,8 +17,10 @@ package initializer import ( "context" "fmt" + "sort" "time" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -93,9 +95,22 @@ func (r *LogStorageConditions) Reconcile(ctx context.Context, request reconcile. } // Compare and update the current StatusCondition if there are any new changes - ls.Status.Conditions = updateConditions(currentConditions, desiredConditions) + conditions := updateConditions(currentConditions, desiredConditions) + + // Skip the write if nothing changed. This controller watches LogStorage, so a no-op write would + // re-trigger it and spin: each reconcile would bump the resourceVersion and enqueue another one. + if equality.Semantic.DeepEqual(ls.Status.Conditions, conditions) { + return reconcile.Result{}, nil + } + ls.Status.Conditions = conditions if err := r.client.Status().Update(ctx, ls); err != nil { + if errors.IsConflict(err) { + // The LogStorage was modified after we read it - our cached copy is stale. Requeue and + // recompute the conditions from the updated object instead of reporting an error. + reqLogger.V(1).Info("Conflict updating LogStorage status conditions, retrying") + return reconcile.Result{Requeue: true}, nil + } log.WithValues("reason", err).Info("Failed to update LogStorage status conditions") return reconcile.Result{}, err } @@ -206,5 +221,12 @@ func updateConditions(currentConditions, desiredConditions map[string]metav1.Con statusConditions = append(statusConditions, desired) } + + // desiredConditions is a map, so iteration order is random. Sort by type to keep the stored + // conditions stable across reconciles - otherwise every write reorders the list, which counts + // as a change and triggers another reconcile. + sort.Slice(statusConditions, func(i, j int) bool { + return statusConditions[i].Type < statusConditions[j].Type + }) return statusConditions } From 96dc12329670199a830cf61bc9b438abc1d44198 Mon Sep 17 00:00:00 2001 From: Alina Militaru <41362174+asincu@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:16:47 -0700 Subject: [PATCH 4/6] (fix): Use CalicoEnterprise instead of TigeraEnterprise --- pkg/render/kubecontrollers/kube-controllers_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index f103f28ba9..3b9d7cc31c 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -568,7 +568,7 @@ var _ = Describe("kube-controllers rendering tests", func() { }) It("should not enable the elasticsearchconfiguration controller when migrating to single-index storage", func() { - instance.Variant = operatorv1.TigeraSecureEnterprise + instance.Variant = operatorv1.CalicoEnterprise cfg.LogStorageExists = true cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 From cba9c2392bc85aecd0496b5da240ee6258aac15b Mon Sep 17 00:00:00 2001 From: Alina Militaru <41362174+asincu@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:10:23 -0700 Subject: [PATCH 5/6] (logstorage-users): Provision Elasticsearch users for all single tenant clusters Calico Cloud single-tenant clusters had their Elasticsearch users provisioned by es-kube-controllers, and the operator only took over while migrating to single-index storage. Take over for all of them, and grant Linseed access to the indices its cluster actually stores data in rather than to both name formats. - Run the log-storage users controller for every Calico Cloud single-tenant cluster, and stop es-kube-controllers running its elasticsearch configuration controller there so that it does not overwrite the users we own. Wait for the cloud config ConfigMap rather than erroring when it is not there yet. - Derive the Linseed role's index privileges from the tenant: the declared base index names when the cluster stores data in single-index format, the multi-index names otherwise - dropping the tenant qualifier for clusters on their own Elasticsearch, whose indices do not carry it. - Report the users TigeraStatus in the log-storage conditions aggregate for Calico Cloud rather than only while migrating. - Rename the index migration knob to USE_SINGLE_INDEX / UseSingleIndex, matching the naming already used by the linseed controller and render code. - Move CloudStandardIndices out of the API module and unexport it, as it is only consumed when building the single-tenant Tenant from the cloud config. --- api/v1/tenant_types.go | 17 ------ cmd/main.go | 6 +- pkg/common/discovery/discovery.go | 6 +- .../initializer/conditions_controller.go | 18 +++--- .../logstorage/kubecontrollers/cloud.go | 5 -- .../kubecontrollers/es_kube_controllers.go | 2 - .../logstorage/linseed/linseed_controller.go | 2 +- .../logstorage/users/users_controller.go | 37 +++++++------ .../logstorage/users/users_controller_test.go | 41 ++++++++++---- pkg/controller/options/options.go | 6 +- pkg/controller/utils/elasticsearch.go | 55 +++++++++++++++---- pkg/controller/utils/utils_test.go | 44 ++++++++++++++- pkg/render/common/cloudconfig/cloudconfig.go | 21 ++++++- .../common/cloudconfig/cloudconfig_test.go | 2 +- .../kubecontrollers/kube-controllers.go | 14 ++--- .../kubecontrollers/kube-controllers_test.go | 4 +- 16 files changed, 186 insertions(+), 94 deletions(-) diff --git a/api/v1/tenant_types.go b/api/v1/tenant_types.go index 2ea355ef09..89f2297df6 100644 --- a/api/v1/tenant_types.go +++ b/api/v1/tenant_types.go @@ -67,23 +67,6 @@ var DataTypes = map[DataType]string{ DataTypePolicyActivity: "ELASTIC_POLICY_ACTIVITY_BASE_INDEX_NAME", } -var CloudStandardIndices = map[DataType]string{ - DataTypeAlerts: "calico_alerts_standard", - DataTypeAuditLogs: "calico_auditlogs_standard", - DataTypeBGPLogs: "calico_bgplogs_standard", - DataTypeComplianceBenchmarks: "calico_compliance_benchmarks_results_standard", - DataTypeComplianceReports: "calico_compliance_reports_standard", - DataTypeComplianceSnapshots: "calico_compliance_snapshots_standard", - DataTypeDNSLogs: "calico_dnslogs_standard", - DataTypeFlowLogs: "calico_flowlogs_standard", - DataTypeL7Logs: "calico_l7logs_standard", - DataTypeRuntimeReports: "calico_runtime_reports_standard", - DataTypeThreatFeedsDomainSet: "calico_threatfeeds_domainnameset_standard", - DataTypeThreatFeedsIPSet: "calico_threatfeeds_ipset_standard", - DataTypeWAFLogs: "calico_waflogs_standard", - DataTypePolicyActivity: "calico_policy_activity_standard", -} - type TenantSpec struct { // ID is the unique identifier for this tenant. // +required diff --git a/cmd/main.go b/cmd/main.go index 0bf6be2e51..7470c49154 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -585,12 +585,12 @@ admission policy installation; once an Installation exists it is the authority o } elasticIsMigrating := false - indexIsMigrating := false + useSingleIndex := false useExternalElastic := discovery.UseExternalElastic(bootConfig) if isCloudBuild() { elasticIsMigrating = discovery.ElasticIsMigrating(bootConfig) - indexIsMigrating = discovery.IndexIsMigrating(bootConfig) + useSingleIndex = discovery.UseSingleIndex(bootConfig) if !elasticIsMigrating { if err := verifyElasticSearch(ctx, cs, useExternalElastic); err != nil { setupLog.Error(err, "Elasticsearch configuration verification failed") @@ -623,7 +623,7 @@ admission policy installation; once an Installation exists it is the authority o ElasticExternal: useExternalElastic, Cloud: isCloudBuild(), ESMigration: elasticIsMigrating, - IndexMigration: indexIsMigrating, + UseSingleIndex: useSingleIndex, UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, } diff --git a/pkg/common/discovery/discovery.go b/pkg/common/discovery/discovery.go index e6b77b819b..4ca0e013cf 100644 --- a/pkg/common/discovery/discovery.go +++ b/pkg/common/discovery/discovery.go @@ -307,15 +307,15 @@ func ElasticIsMigrating(config *corev1.ConfigMap) bool { return false } -// IndexIsMigrating returns true if this cluster is in the last phase of a migration to single-index +// UseSingleIndex returns true if this cluster is in the last phase of a migration to single-index // storage, during which the operator must reconfigure Linseed to use the single-index names. -func IndexIsMigrating(config *corev1.ConfigMap) bool { +func UseSingleIndex(config *corev1.ConfigMap) bool { if config == nil { return false } // Load the operator bootstrap configuration from its configmap. - if val, ok := config.Data["INDEX_MIGRATION"]; ok && val != "" { + if val, ok := config.Data["USE_SINGLE_INDEX"]; ok && val != "" { if strings.ToLower(val) == "true" { return true } diff --git a/pkg/controller/logstorage/initializer/conditions_controller.go b/pkg/controller/logstorage/initializer/conditions_controller.go index e35b953acd..7552f431fa 100644 --- a/pkg/controller/logstorage/initializer/conditions_controller.go +++ b/pkg/controller/logstorage/initializer/conditions_controller.go @@ -43,10 +43,10 @@ func AddConditionsController(mgr manager.Manager, opts options.ControllerOptions // Create the reconciler r := &LogStorageConditions{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - multiTenant: opts.MultiTenant, - indexMigration: opts.IndexMigration, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + multiTenant: opts.MultiTenant, + cloud: opts.Cloud, } return ctrl.NewControllerManagedBy(mgr). @@ -66,9 +66,9 @@ type LogStorageConditions struct { scheme *runtime.Scheme multiTenant bool - // indexMigration indicates that this single-tenant cluster is migrating to single-index storage, - // in which case the log-storage users controller runs and reports status. - indexMigration bool + // cloud indicates that this is a Calico Cloud install, in which case the log-storage users + // controller runs in single-tenant mode too and reports status. + cloud bool } func (r *LogStorageConditions) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { @@ -132,8 +132,8 @@ func (r *LogStorageConditions) getDesiredConditions(ctx context.Context) (map[st expectedInstances = append(expectedInstances, TigeraStatusLogStorageUsers) } else { expectedInstances = append(expectedInstances, TigeraStatusLogStorageESMetrics, TigeraStatusLogStorageKubeController, TigeraStatusLogStorageDashboards) - if r.indexMigration { - // While migrating to single-index storage, the users controller runs in single-tenant mode too. + if r.cloud { + // In Calico Cloud, the users controller runs in single-tenant mode too. expectedInstances = append(expectedInstances, TigeraStatusLogStorageUsers) } } diff --git a/pkg/controller/logstorage/kubecontrollers/cloud.go b/pkg/controller/logstorage/kubecontrollers/cloud.go index 19a183f95b..93a4c4110d 100644 --- a/pkg/controller/logstorage/kubecontrollers/cloud.go +++ b/pkg/controller/logstorage/kubecontrollers/cloud.go @@ -70,11 +70,6 @@ func (r *ESKubeControllersController) esGatewayAddCloudModificationsToConfig(c * // esKubeControllersAddCloudModificationsToConfig modifies the provided *kubecontrollers.KubeControllersConfiguration to include Calico Cloud specific configuration. func (r *ESKubeControllersController) esKubeControllersAddCloudModificationsToConfig(c *kubecontrollers.KubeControllersConfiguration, reqLogger logr.Logger, ctx context.Context) (reconcile.Result, bool, error) { - // While migrating to single-index storage, es-kube-controllers must not run the elasticsearch - // configuration controller. The operator's log-storage users controller provisions the - // Elasticsearch users instead, so that Linseed gets the RBAC needed for the new indices. - c.IndexMigration = r.indexMigration - if r.cloud && r.elasticExternal && !r.multiTenant { cloudConfig, err := utils.GetCloudConfig(ctx, r.client) if err != nil { diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index cd1e227306..735685963c 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -64,7 +64,6 @@ type ESKubeControllersController struct { elasticExternal bool multiTenant bool cloud bool - indexMigration bool tierWatchReady *utils.ReadyFlag } @@ -91,7 +90,6 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { elasticExternal: opts.ElasticExternal, multiTenant: opts.MultiTenant, cloud: opts.Cloud, - indexMigration: opts.IndexMigration, tierWatchReady: &utils.ReadyFlag{}, } r.status.Run(opts.ShutdownContext) diff --git a/pkg/controller/logstorage/linseed/linseed_controller.go b/pkg/controller/logstorage/linseed/linseed_controller.go index d780b090ae..be405abd32 100644 --- a/pkg/controller/logstorage/linseed/linseed_controller.go +++ b/pkg/controller/logstorage/linseed/linseed_controller.go @@ -90,7 +90,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { status: status.New(mgr.GetClient(), "log-storage-access", opts.KubernetesVersion), elasticExternal: opts.ElasticExternal, cloud: opts.Cloud, - useSingleIndex: opts.IndexMigration, + useSingleIndex: opts.UseSingleIndex, } r.status.Run(opts.ShutdownContext) diff --git a/pkg/controller/logstorage/users/users_controller.go b/pkg/controller/logstorage/users/users_controller.go index 3018c341e9..e637f694b4 100644 --- a/pkg/controller/logstorage/users/users_controller.go +++ b/pkg/controller/logstorage/users/users_controller.go @@ -62,10 +62,10 @@ type UserController struct { multiTenant bool elasticExternal bool - // indexMigration indicates that this single-tenant cluster is migrating to single-index storage. - // While migrating, this controller provisions the Elasticsearch users instead of es-kube-controllers, - // so that Linseed gets the RBAC needed for the new indices. - indexMigration bool + // useSingleIndex indicates that this single-tenant cluster stores its data in single-index format, + // in which case the users provisioned here are granted access to the single-index names rather than + // to the per-cluster multi-index ones. + useSingleIndex bool } type UsersCleanupController struct { @@ -79,10 +79,10 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if !opts.Variant.IsEnterprise() { return nil } - if !opts.MultiTenant && !opts.IndexMigration { - // The operator only creates users in multi-tenant mode, or in single-tenant mode while - // migrating to single-index storage. Otherwise, user creation is handled by - // es-kube-controllers instead. + if !opts.MultiTenant && !opts.Cloud { + // The operator creates users for multi-tenant clusters, and for the single-tenant clusters of + // Calico Cloud - which build their tenant configuration from the cloud config ConfigMap. Anywhere + // else, user creation is handled by es-kube-controllers instead. return nil } @@ -91,7 +91,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { client: mgr.GetClient(), scheme: mgr.GetScheme(), multiTenant: opts.MultiTenant, - indexMigration: opts.IndexMigration, + useSingleIndex: opts.UseSingleIndex, status: status.New(mgr.GetClient(), initializer.TigeraStatusLogStorageUsers, opts.KubernetesVersion), esClientFn: utils.NewElasticClient, elasticExternal: opts.ElasticExternal, @@ -203,11 +203,16 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Single-tenant clusters have no Tenant resource. Build the equivalent tenant configuration // from the cloud config, so that we provision the users against the right Elasticsearch. cloudConfig, err := utils.GetCloudConfig(ctx, r.client) - if err != nil { + if errors.IsNotFound(err) { + // The cloud config is written out of band, and may not exist yet. Wait for it rather than + // retrying with backoff - we watch the ConfigMap, so we reconcile again once it appears. + r.status.SetDegraded(operatorv1.ResourceNotReady, fmt.Sprintf("Waiting for ConfigMap %s to be available", cloudconfig.CloudConfigConfigMapName), nil, reqLogger) + return reconcile.Result{}, nil + } else if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant(r.indexMigration) + tenant = cloudConfig.ToTenant(r.useSingleIndex) tenantID = tenant.Spec.ID } @@ -257,10 +262,10 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques if err != nil { return reconcile.Result{}, err } - linseedUser = utils.LinseedUser(clusterID, tenantID) + linseedUser = utils.LinseedUser(clusterID, tenant) dashboardUser = utils.DashboardUser(clusterID, tenantID) } else { - linseedUser = utils.LinseedUserSingleTenant(tenantID) + linseedUser = utils.LinseedUserSingleTenant(tenant, r.elasticExternal) dashboardUser = utils.DashboardUserSingleTenant(tenantID) } @@ -282,8 +287,8 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques credentialSecrets = append(credentialSecrets, &linseedUserSecret) } else if string(linseedUserSecret.Data["username"]) != linseedUser.Username { // The credentials exist, but reference a different Elasticsearch user than the one we provision - - // e.g. because they were created by es-kube-controllers before this cluster started migrating to - // single-index storage. Point them at our user, keeping the existing password. + // e.g. because they were created by es-kube-controllers before the operator took over user + // provisioning. Point them at our user, keeping the existing password. linseedUserSecret.StringData = map[string]string{"username": linseedUser.Username} credentialSecrets = append(credentialSecrets, &linseedUserSecret) } @@ -481,7 +486,7 @@ func (r *UsersCleanupController) cleanupStaleUsers(ctx context.Context, logger l return fmt.Errorf("failed to fetch users from Elasticsearch") } - lu := utils.LinseedUser(clusterID, t.Spec.ID) + lu := utils.LinseedUser(clusterID, &t) dashboardsUser := utils.DashboardUser(clusterID, t.Spec.ID) for _, user := range allESUsers { if user.Username == lu.Username || user.Username == dashboardsUser.Username { diff --git a/pkg/controller/logstorage/users/users_controller_test.go b/pkg/controller/logstorage/users/users_controller_test.go index 70605f8228..028f0424f9 100644 --- a/pkg/controller/logstorage/users/users_controller_test.go +++ b/pkg/controller/logstorage/users/users_controller_test.go @@ -69,17 +69,20 @@ var _ = Describe("LogStorage cleanup controller", func() { tenantID1 := "tenant1" tenantID2 := "tenant2" - staleLinseedUser := utils.LinseedUser(clusterID1, tenantID1) + tenant1 := &operatorv1.Tenant{Spec: operatorv1.TenantSpec{ID: tenantID1}} + tenant2 := &operatorv1.Tenant{Spec: operatorv1.TenantSpec{ID: tenantID2}} + + staleLinseedUser := utils.LinseedUser(clusterID1, tenant1) staleDashboardsUser := utils.DashboardUser(clusterID1, tenantID1) esTestUsers := []utils.User{ *staleLinseedUser, *staleDashboardsUser, - *utils.LinseedUser(clusterID1, tenantID2), + *utils.LinseedUser(clusterID1, tenant2), *utils.DashboardUser(clusterID1, tenantID2), - *utils.LinseedUser(clusterID2, tenantID1), + *utils.LinseedUser(clusterID2, tenant1), *utils.DashboardUser(clusterID2, tenantID1), - *utils.LinseedUser(clusterID2, tenantID2), + *utils.LinseedUser(clusterID2, tenant2), *utils.DashboardUser(clusterID2, tenantID2), } @@ -183,10 +186,15 @@ var _ = Describe("LogStorage users controller", func() { }, multiTenant: false, elasticExternal: true, - indexMigration: true, + useSingleIndex: true, } }) + // singleTenant builds the Tenant that the controller derives from the cloud config ConfigMap created above. + singleTenant := func(useSingleIndex bool) *operatorv1.Tenant { + return cloudconfig.NewCloudConfig(tenantID, "tenant-a-name", "es.example.com", "kb.example.com", false).ToTenant(useSingleIndex) + } + // secretValue reads a value from a Secret. The fake client doesn't convert StringData into Data // the way the API server does, so we may find the value in either field. secretValue := func(name, namespace, key string) string { @@ -198,19 +206,20 @@ var _ = Describe("LogStorage users controller", func() { return string(s.Data[key]) } - It("should provision users for a single-tenant cluster migrating to single-index storage", func() { + It("should provision users for a single-tenant cluster using single-index storage", func() { _, err := r.Reconcile(ctx, reconcile.Request{}) Expect(err).NotTo(HaveOccurred()) // The Linseed user should be created in ES with the name es-kube-controllers used, and with - // privileges on the new single-index calico_* indices. - expected := utils.LinseedUserSingleTenant(tenantID) + // privileges on the single-index calico_* indices. + expected := utils.LinseedUserSingleTenant(singleTenant(true), true) Expect(expected.Username).To(Equal("tigera-ee-linseed-tenant-a-secure")) Expect(esClient.created).To(HaveLen(2)) Expect(esClient.created[0].Username).To(Equal(expected.Username)) Expect(esClient.created[0].Roles).To(Equal(expected.Roles)) Expect(esClient.created[0].Roles[0].Name).To(Equal(expected.Username)) - Expect(esClient.created[0].Roles[0].Definition.Indices[0].Names).To(ContainElement("calico_*")) + Expect(esClient.created[0].Roles[0].Definition.Indices[0].Names).To(ContainElement("calico_flowlogs_standard*")) + Expect(esClient.created[0].Roles[0].Definition.Indices[0].Names).To(HaveLen(len(operatorv1.DataTypes))) Expect(esClient.created[1].Username).To(Equal("tigera-ee-dashboards-installer-tenant-a-secure")) // The credentials should be written to the Elasticsearch namespace for Linseed to consume. @@ -229,7 +238,7 @@ var _ = Describe("LogStorage users controller", func() { _, err := r.Reconcile(ctx, reconcile.Request{}) Expect(err).NotTo(HaveOccurred()) - expected := utils.LinseedUserSingleTenant(tenantID) + expected := utils.LinseedUserSingleTenant(singleTenant(true), true) Expect(secretValue(render.ElasticsearchLinseedUserSecret, common.OperatorNamespace(), "username")).To(Equal(expected.Username)) Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "username")).To(Equal(expected.Username)) @@ -238,4 +247,16 @@ var _ = Describe("LogStorage users controller", func() { Expect(esClient.created[0].Username).To(Equal(expected.Username)) Expect(esClient.created[0].Password).To(Equal("existing-password")) }) + + It("should grant the Linseed user the multi-index names when not using single-index storage", func() { + r.useSingleIndex = false + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + + // The user keeps its name, but is only granted access to this tenant's multi-index format indices. + expected := utils.LinseedUserSingleTenant(singleTenant(false), true) + Expect(esClient.created[0].Username).To(Equal(expected.Username)) + Expect(esClient.created[0].Roles[0].Definition.Indices[0].Names).To(Equal([]string{"tigera_secure_ee_*.tenant-a.*.*"})) + }) }) diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 15bf9e7d9f..3bcf5276a7 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -61,9 +61,9 @@ type ControllerOptions struct { // LSS configuration and internal elasticsearch running. Only meaningful when Cloud is set. ESMigration bool - // This field is enabled in the last phase of an index migration for a single tenant format, - // we need to reconfigure - IndexMigration bool + // UseSingleIndex is enabled in the last phase of an index migration for a single tenant cluster, + // during which the operator reconfigures log storage to use the single-index names. + UseSingleIndex bool // Whether or not to use crd.projectcalico.org/v1 or projectcalico.org/v3 for Calico CRDs. UseV3CRDs bool diff --git a/pkg/controller/utils/elasticsearch.go b/pkg/controller/utils/elasticsearch.go index bbe4cde489..692a1dee01 100644 --- a/pkg/controller/utils/elasticsearch.go +++ b/pkg/controller/utils/elasticsearch.go @@ -232,18 +232,54 @@ func formatNameSingleTenant(name, tenantID string) string { return fmt.Sprintf("%s-%s-%s", name, tenantID, ElasticsearchSecureUserSuffix) } -func LinseedUser(clusterID, tenant string) *User { - return linseedUser(formatName(ElasticsearchUserNameLinseed, clusterID, tenant), tenant) +// LinseedUser returns the Linseed user for a multi-tenant cluster. Multi-tenant clusters always store +// their data in single-index format, so the user is granted access to the indices declared on the Tenant. +func LinseedUser(clusterID string, tenant *operatorv1.Tenant) *User { + return linseedUser(formatName(ElasticsearchUserNameLinseed, clusterID, tenant.Spec.ID), singleIndexNames(tenant)) } -// LinseedUserSingleTenant returns the Linseed user for a single-tenant cluster. It carries the same -// privileges as the multi-tenant user - including access to the single-index calico_* indices - but is -// named the way es-kube-controllers named it, and needs no cluster ID. -func LinseedUserSingleTenant(tenant string) *User { - return linseedUser(formatNameSingleTenant(ElasticsearchUserNameLinseed, tenant), tenant) +// LinseedUserSingleTenant returns the Linseed user for a single-tenant cluster. It is named the way +// es-kube-controllers named it, and needs no cluster ID. +// +// A single-tenant cluster only declares indices on its Tenant once it has moved to single-index storage; +// until then its data lives in the per-cluster multi-index format, and the user is granted access to +// that instead. +func LinseedUserSingleTenant(tenant *operatorv1.Tenant, externalElastic bool) *User { + names := multiIndexNames(tenant.Spec.ID, externalElastic) + if len(tenant.Spec.Indices) > 0 { + names = singleIndexNames(tenant) + } + return linseedUser(formatNameSingleTenant(ElasticsearchUserNameLinseed, tenant.Spec.ID), names) +} + +// singleIndexNames returns the index patterns covering the tenant's single-index storage. Each declared +// base index name is wildcarded so that the pattern matches the write alias as well as the numbered +// indices behind it. Tenants that declare no indices leave Linseed on its default index names, which are +// all prefixed with calico_. +func singleIndexNames(tenant *operatorv1.Tenant) []string { + if len(tenant.Spec.Indices) == 0 { + return []string{"calico_*"} + } + + names := make([]string, 0, len(tenant.Spec.Indices)) + for _, index := range tenant.Spec.Indices { + names = append(names, fmt.Sprintf("%s*", index.BaseIndexName)) + } + return names +} + +// multiIndexNames returns the index pattern covering multi-index storage, where every cluster writes to +// its own set of indices. Indices written to an Elasticsearch shared between tenants carry the tenant ID +// in their name, and the pattern is scoped to it. Indices written to the cluster's own Elasticsearch do +// not - Linseed has its tenant suffix disabled there - so there is nothing to scope the pattern to. +func multiIndexNames(tenant string, externalElastic bool) []string { + if !externalElastic { + tenant = "" + } + return []string{indexPattern("tigera_secure_ee_*", "*", ".*", tenant)} } -func linseedUser(username, tenant string) *User { +func linseedUser(username string, indices []string) *User { return &User{ Username: username, Roles: []Role{ @@ -253,8 +289,7 @@ func linseedUser(username, tenant string) *User { Cluster: []string{"monitor", "manage_index_templates", "manage_ilm"}, Indices: []RoleIndex{ { - // Include both single-index and multi-index name formats. - Names: []string{indexPattern("tigera_secure_ee_*", "*", ".*", tenant), "calico_*"}, + Names: indices, Privileges: []string{"create_index", "write", "manage", "read"}, }, }, diff --git a/pkg/controller/utils/utils_test.go b/pkg/controller/utils/utils_test.go index ecc9a33a1a..1207a4ada7 100644 --- a/pkg/controller/utils/utils_test.go +++ b/pkg/controller/utils/utils_test.go @@ -291,7 +291,16 @@ var _ = Describe("Utils ElasticSearch test", func() { }) It("should generate Linseed ElasticUser with expected username and roles", func() { - linseedUser := LinseedUser(clusterID, tenantID) + tenant := &opv1.Tenant{ + Spec: opv1.TenantSpec{ + ID: tenantID, + Indices: []opv1.Index{ + {DataType: opv1.DataTypeFlowLogs, BaseIndexName: "calico_flowlogs_standard"}, + {DataType: opv1.DataTypeDNSLogs, BaseIndexName: "calico_dnslogs_standard"}, + }, + }, + } + linseedUser := LinseedUser(clusterID, tenant) expectedLinseedESName := fmt.Sprintf("%s_%s_%s", ElasticsearchUserNameLinseed, clusterID, tenantID) Expect(linseedUser.Username).To(Equal(expectedLinseedESName)) @@ -303,8 +312,8 @@ var _ = Describe("Utils ElasticSearch test", func() { Cluster: []string{"monitor", "manage_index_templates", "manage_ilm"}, Indices: []RoleIndex{ { - // Include both single-index and multi-index name formats. - Names: []string{indexPattern("tigera_secure_ee_*", "*", ".*", tenantID), "calico_*"}, + // The indices declared on the Tenant, wildcarded to cover the indices behind each alias. + Names: []string{"calico_flowlogs_standard*", "calico_dnslogs_standard*"}, Privileges: []string{"create_index", "write", "manage", "read"}, }, }, @@ -312,6 +321,35 @@ var _ = Describe("Utils ElasticSearch test", func() { Expect(*linseedRole.Definition).To(Equal(expectedLinseedRoleDef)) }) + + It("should grant a multi-tenant Linseed user the default single-index names when the Tenant declares none", func() { + tenant := &opv1.Tenant{Spec: opv1.TenantSpec{ID: tenantID}} + linseedUser := LinseedUser(clusterID, tenant) + + Expect(linseedUser.Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_*"})) + }) + + It("should generate a single-tenant Linseed user granted the indices its cluster stores data in", func() { + tenant := &opv1.Tenant{Spec: opv1.TenantSpec{ID: tenantID}} + expectedName := fmt.Sprintf("%s-%s-%s", ElasticsearchUserNameLinseed, tenantID, ElasticsearchSecureUserSuffix) + + // A cluster still on multi-index storage declares no indices, and gets access to the indices + // named for its tenant. + linseedUser := LinseedUserSingleTenant(tenant, true) + Expect(linseedUser.Username).To(Equal(expectedName)) + Expect(linseedUser.Roles[0].Name).To(Equal(expectedName)) + Expect(linseedUser.Roles[0].Definition.Indices[0].Names).To(Equal([]string{indexPattern("tigera_secure_ee_*", "*", ".*", tenantID)})) + + // Indices in the cluster's own Elasticsearch are not qualified by tenant, so neither is the pattern. + linseedUser = LinseedUserSingleTenant(tenant, false) + Expect(linseedUser.Roles[0].Definition.Indices[0].Names).To(Equal([]string{indexPattern("tigera_secure_ee_*", "*", ".*", "")})) + + // Once it moves to single-index storage, it gets access to the declared indices instead. + tenant.Spec.Indices = []opv1.Index{{DataType: opv1.DataTypeFlowLogs, BaseIndexName: "calico_flowlogs_standard"}} + linseedUser = LinseedUserSingleTenant(tenant, true) + Expect(linseedUser.Username).To(Equal(expectedName)) + Expect(linseedUser.Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_flowlogs_standard*"})) + }) }) type fakeClient struct { diff --git a/pkg/render/common/cloudconfig/cloudconfig.go b/pkg/render/common/cloudconfig/cloudconfig.go index 7a5a530529..a719194c97 100644 --- a/pkg/render/common/cloudconfig/cloudconfig.go +++ b/pkg/render/common/cloudconfig/cloudconfig.go @@ -31,6 +31,25 @@ const ( CloudConfigConfigMapName = "tigera-secure-cloud-config" ) +// cloudStandardIndices maps each data type to the standard index base name used by clusters that +// have migrated to single-index storage. +var cloudStandardIndices = map[v1.DataType]string{ + v1.DataTypeAlerts: "calico_alerts_standard", + v1.DataTypeAuditLogs: "calico_auditlogs_standard", + v1.DataTypeBGPLogs: "calico_bgplogs_standard", + v1.DataTypeComplianceBenchmarks: "calico_compliance_benchmarks_results_standard", + v1.DataTypeComplianceReports: "calico_compliance_reports_standard", + v1.DataTypeComplianceSnapshots: "calico_compliance_snapshots_standard", + v1.DataTypeDNSLogs: "calico_dnslogs_standard", + v1.DataTypeFlowLogs: "calico_flowlogs_standard", + v1.DataTypeL7Logs: "calico_l7logs_standard", + v1.DataTypeRuntimeReports: "calico_runtime_reports_standard", + v1.DataTypeThreatFeedsDomainSet: "calico_threatfeeds_domainnameset_standard", + v1.DataTypeThreatFeedsIPSet: "calico_threatfeeds_ipset_standard", + v1.DataTypeWAFLogs: "calico_waflogs_standard", + v1.DataTypePolicyActivity: "calico_policy_activity_standard", +} + func NewCloudConfig(tenantId string, tenantName string, externalESDomain string, externalKibanaDomain string, enableMTLS bool) *CloudConfig { return &CloudConfig{ tenantId: tenantId, @@ -109,7 +128,7 @@ func (c CloudConfig) ToTenant(useSingleIndex bool) *v1.Tenant { } for dataType := range v1.DataTypes { - tenant.Spec.Indices = append(tenant.Spec.Indices, v1.Index{DataType: dataType, BaseIndexName: v1.CloudStandardIndices[dataType]}) + tenant.Spec.Indices = append(tenant.Spec.Indices, v1.Index{DataType: dataType, BaseIndexName: cloudStandardIndices[dataType]}) } // DataTypes is a map, so iteration order is random. Sort by data type to keep the generated diff --git a/pkg/render/common/cloudconfig/cloudconfig_test.go b/pkg/render/common/cloudconfig/cloudconfig_test.go index f8d8d5f23d..b2e41d04ed 100644 --- a/pkg/render/common/cloudconfig/cloudconfig_test.go +++ b/pkg/render/common/cloudconfig/cloudconfig_test.go @@ -125,7 +125,7 @@ var _ = Describe("CloudConfig ConfigMap tests", func() { indices := cloudConfig.ToTenant(true).Spec.Indices Expect(indices).To(HaveLen(len(v1.DataTypes))) for _, index := range indices { - Expect(index.BaseIndexName).To(Equal(v1.CloudStandardIndices[index.DataType])) + Expect(index.BaseIndexName).To(Equal(cloudStandardIndices[index.DataType])) Expect(index.BaseIndexName).ToNot(BeEmpty()) } }) diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index ea9e1d6dd8..980bd16db9 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -122,13 +122,10 @@ type KubeControllersConfiguration struct { TenantID string // Cloud indicates kube-controllers is being rendered for a Calico Cloud install. When false the - // cloud-specific RBAC below is not granted and enterprise RBAC is unchanged. - Cloud bool - - // IndexMigration indicates that this cluster is migrating to single-index storage. While migrating, + // cloud-specific RBAC below is not granted and enterprise RBAC is unchanged. In Calico Cloud, // Elasticsearch configuration - including provisioning of the Linseed user - is handled by the // operator's log-storage users controller rather than by es-kube-controllers. - IndexMigration bool + Cloud bool MetricsServerTLS certificatemanagement.KeyPairInterface @@ -288,9 +285,10 @@ func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeCon if !cfg.Tenant.MultiTenant() { // Zero and single tenant cluster needs elasticsearch configuration enabledControllers = append(enabledControllers, "authorization") - if !cfg.IndexMigration { - // While migrating to single-index storage, the operator's log-storage users controller - // provisions the Elasticsearch users so that they get the RBAC needed for the new indices. + if !cfg.Cloud { + // In Calico Cloud, the operator's log-storage users controller provisions the Elasticsearch + // users itself, so that they get the RBAC for the indices the cluster actually stores its + // data in. Running this controller as well would have it overwrite those users. enabledControllers = append(enabledControllers, "elasticsearchconfiguration") } if cfg.ManagementCluster != nil && cfg.Tenant == nil { diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 3b9d7cc31c..8e82da5764 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -567,12 +567,12 @@ var _ = Describe("kube-controllers rendering tests", func() { })) }) - It("should not enable the elasticsearchconfiguration controller when migrating to single-index storage", func() { + It("should not enable the elasticsearchconfiguration controller in Calico Cloud", func() { instance.Variant = operatorv1.CalicoEnterprise cfg.LogStorageExists = true cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret cfg.MetricsPort = 9094 - cfg.IndexMigration = true + cfg.Cloud = true component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) From 5081d5f7ed52db926cd0ed4658cf44f93b4ee2a9 Mon Sep 17 00:00:00 2001 From: Alina Militaru <41362174+asincu@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:51:00 -0700 Subject: [PATCH 6/6] (logstorage-users): Address review feedback on Linseed index privileges - Skip indices with an empty base index name when building the Linseed role's index privileges. A Tenant declaring an index without a base name - whether misconfigured, or carrying a DataType added later without a mapping - would otherwise be wildcarded into "*", granting Linseed access to every index in Elasticsearch. Fall back to Linseed's default calico_ names when no usable base index name remains, which is what a Tenant declaring no indices at all already got. - Fix a comment on the single-tenant Linseed backend, which described the branch as migrating to multi-tenant style indices when it is gated by UseSingleIndex and migrating to single-index storage. Co-Authored-By: Claude Opus 5 (1M context) --- pkg/controller/utils/elasticsearch.go | 17 +++++++++++------ pkg/controller/utils/utils_test.go | 21 +++++++++++++++++++++ pkg/render/logstorage/linseed/linseed.go | 2 +- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/pkg/controller/utils/elasticsearch.go b/pkg/controller/utils/elasticsearch.go index 692a1dee01..875d1857fe 100644 --- a/pkg/controller/utils/elasticsearch.go +++ b/pkg/controller/utils/elasticsearch.go @@ -254,17 +254,22 @@ func LinseedUserSingleTenant(tenant *operatorv1.Tenant, externalElastic bool) *U // singleIndexNames returns the index patterns covering the tenant's single-index storage. Each declared // base index name is wildcarded so that the pattern matches the write alias as well as the numbered -// indices behind it. Tenants that declare no indices leave Linseed on its default index names, which are -// all prefixed with calico_. +// indices behind it. Indices without a base index name are skipped, so that a misconfigured Tenant +// cannot widen the pattern to "*" and grant access to every index in Elasticsearch. Tenants that +// declare no usable indices leave Linseed on its default index names, which are all prefixed with +// calico_. func singleIndexNames(tenant *operatorv1.Tenant) []string { - if len(tenant.Spec.Indices) == 0 { - return []string{"calico_*"} - } - names := make([]string, 0, len(tenant.Spec.Indices)) for _, index := range tenant.Spec.Indices { + if index.BaseIndexName == "" { + continue + } names = append(names, fmt.Sprintf("%s*", index.BaseIndexName)) } + + if len(names) == 0 { + return []string{"calico_*"} + } return names } diff --git a/pkg/controller/utils/utils_test.go b/pkg/controller/utils/utils_test.go index 1207a4ada7..ffd83cddb4 100644 --- a/pkg/controller/utils/utils_test.go +++ b/pkg/controller/utils/utils_test.go @@ -350,6 +350,27 @@ var _ = Describe("Utils ElasticSearch test", func() { Expect(linseedUser.Username).To(Equal(expectedName)) Expect(linseedUser.Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_flowlogs_standard*"})) }) + + It("should never widen the single-index pattern to all indices when a base index name is empty", func() { + // An index with no base index name must not be wildcarded into "*", which would grant Linseed + // access to every index in Elasticsearch. + tenant := &opv1.Tenant{ + Spec: opv1.TenantSpec{ + ID: tenantID, + Indices: []opv1.Index{ + {DataType: opv1.DataTypeFlowLogs, BaseIndexName: "calico_flowlogs_standard"}, + {DataType: opv1.DataTypeDNSLogs}, + }, + }, + } + Expect(LinseedUser(clusterID, tenant).Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_flowlogs_standard*"})) + Expect(LinseedUserSingleTenant(tenant, true).Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_flowlogs_standard*"})) + + // With no usable base index name at all, fall back to Linseed's default index names. + tenant.Spec.Indices = []opv1.Index{{DataType: opv1.DataTypeDNSLogs}} + Expect(LinseedUser(clusterID, tenant).Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_*"})) + Expect(LinseedUserSingleTenant(tenant, true).Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_*"})) + }) }) type fakeClient struct { diff --git a/pkg/render/logstorage/linseed/linseed.go b/pkg/render/logstorage/linseed/linseed.go index 66949407a9..6a2af9445c 100644 --- a/pkg/render/logstorage/linseed/linseed.go +++ b/pkg/render/logstorage/linseed/linseed.go @@ -428,7 +428,7 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { replicas = l.cfg.Tenant.Spec.ControlPlaneReplicas } } else if l.cfg.UseSingleIndex { - // For single-tenant clusters migrating to multi-tenant style indices, + // For single-tenant clusters migrating to single-index storage, // use the elastic-single-index backend and configure index base names. envVars = append(envVars, corev1.EnvVar{Name: "BACKEND", Value: "elastic-single-index"}) for _, index := range l.cfg.Tenant.Spec.Indices {