diff --git a/cmd/main.go b/cmd/main.go index a1b648be36..7470c49154 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -585,10 +585,12 @@ admission policy installation; once an Installation exists it is the authority o } elasticIsMigrating := false + useSingleIndex := false useExternalElastic := discovery.UseExternalElastic(bootConfig) if isCloudBuild() { elasticIsMigrating = discovery.ElasticIsMigrating(bootConfig) + useSingleIndex = discovery.UseSingleIndex(bootConfig) if !elasticIsMigrating { if err := verifyElasticSearch(ctx, cs, useExternalElastic); err != nil { setupLog.Error(err, "Elasticsearch configuration verification failed") @@ -621,6 +623,7 @@ admission policy installation; once an Installation exists it is the authority o ElasticExternal: useExternalElastic, Cloud: isCloudBuild(), ESMigration: elasticIsMigrating, + UseSingleIndex: useSingleIndex, UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, } diff --git a/pkg/common/discovery/discovery.go b/pkg/common/discovery/discovery.go index cc0ee2b2b5..4ca0e013cf 100644 --- a/pkg/common/discovery/discovery.go +++ b/pkg/common/discovery/discovery.go @@ -306,3 +306,19 @@ func ElasticIsMigrating(config *corev1.ConfigMap) bool { } return false } + +// UseSingleIndex returns true if this cluster is in the last phase of a migration to single-index +// storage, during which the operator must reconfigure Linseed to use the single-index names. +func UseSingleIndex(config *corev1.ConfigMap) bool { + if config == nil { + return false + } + + // Load the operator bootstrap configuration from its configmap. + if val, ok := config.Data["USE_SINGLE_INDEX"]; ok && val != "" { + if strings.ToLower(val) == "true" { + return true + } + } + return false +} diff --git a/pkg/controller/intrusiondetection/intrusiondetection_controller.go b/pkg/controller/intrusiondetection/intrusiondetection_controller.go index f46915a891..68ce7b538f 100644 --- a/pkg/controller/intrusiondetection/intrusiondetection_controller.go +++ b/pkg/controller/intrusiondetection/intrusiondetection_controller.go @@ -437,7 +437,7 @@ func (r *ReconcileIntrusionDetection) Reconcile(ctx context.Context, request rec r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } // Create a component handler to manage the rendered component. diff --git a/pkg/controller/logstorage/dashboards/dashboards_controller.go b/pkg/controller/logstorage/dashboards/dashboards_controller.go index 9bf035c1d7..eecab837a8 100644 --- a/pkg/controller/logstorage/dashboards/dashboards_controller.go +++ b/pkg/controller/logstorage/dashboards/dashboards_controller.go @@ -290,7 +290,7 @@ func (d DashboardsSubController) Reconcile(ctx context.Context, request reconcil d.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } // Determine the host and port from the URL. @@ -323,7 +323,8 @@ func (d DashboardsSubController) Reconcile(ctx context.Context, request reconcil // Query the username and password this Dashboards Installer instance should use to authenticate with Elasticsearch. // For multi-tenant systems, credentials are created by the elasticsearch users controller. - // For single-tenant system, these are created by es-kube-controllers. + // For single-tenant systems, these are created by es-kube-controllers, unless the cluster is + // migrating to single-index storage - then the users controller creates them as well. key = types.NamespacedName{Name: dashboards.ElasticCredentialsSecret, Namespace: helper.InstallNamespace()} credentials := corev1.Secret{} if err = d.client.Get(ctx, key, &credentials); err != nil && !errors.IsNotFound(err) { diff --git a/pkg/controller/logstorage/initializer/conditions_controller.go b/pkg/controller/logstorage/initializer/conditions_controller.go index 847a589b68..7552f431fa 100644 --- a/pkg/controller/logstorage/initializer/conditions_controller.go +++ b/pkg/controller/logstorage/initializer/conditions_controller.go @@ -17,8 +17,10 @@ package initializer import ( "context" "fmt" + "sort" "time" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -44,6 +46,7 @@ func AddConditionsController(mgr manager.Manager, opts options.ControllerOptions client: mgr.GetClient(), scheme: mgr.GetScheme(), multiTenant: opts.MultiTenant, + cloud: opts.Cloud, } return ctrl.NewControllerManagedBy(mgr). @@ -62,6 +65,10 @@ type LogStorageConditions struct { client client.Client scheme *runtime.Scheme multiTenant bool + + // cloud indicates that this is a Calico Cloud install, in which case the log-storage users + // controller runs in single-tenant mode too and reports status. + cloud bool } func (r *LogStorageConditions) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { @@ -88,9 +95,22 @@ func (r *LogStorageConditions) Reconcile(ctx context.Context, request reconcile. } // Compare and update the current StatusCondition if there are any new changes - ls.Status.Conditions = updateConditions(currentConditions, desiredConditions) + conditions := updateConditions(currentConditions, desiredConditions) + + // Skip the write if nothing changed. This controller watches LogStorage, so a no-op write would + // re-trigger it and spin: each reconcile would bump the resourceVersion and enqueue another one. + if equality.Semantic.DeepEqual(ls.Status.Conditions, conditions) { + return reconcile.Result{}, nil + } + ls.Status.Conditions = conditions if err := r.client.Status().Update(ctx, ls); err != nil { + if errors.IsConflict(err) { + // The LogStorage was modified after we read it - our cached copy is stale. Requeue and + // recompute the conditions from the updated object instead of reporting an error. + reqLogger.V(1).Info("Conflict updating LogStorage status conditions, retrying") + return reconcile.Result{Requeue: true}, nil + } log.WithValues("reason", err).Info("Failed to update LogStorage status conditions") return reconcile.Result{}, err } @@ -112,6 +132,10 @@ func (r *LogStorageConditions) getDesiredConditions(ctx context.Context) (map[st expectedInstances = append(expectedInstances, TigeraStatusLogStorageUsers) } else { expectedInstances = append(expectedInstances, TigeraStatusLogStorageESMetrics, TigeraStatusLogStorageKubeController, TigeraStatusLogStorageDashboards) + if r.cloud { + // In Calico Cloud, the users controller runs in single-tenant mode too. + expectedInstances = append(expectedInstances, TigeraStatusLogStorageUsers) + } } // Keep track of which instances are in which state. @@ -197,5 +221,12 @@ func updateConditions(currentConditions, desiredConditions map[string]metav1.Con statusConditions = append(statusConditions, desired) } + + // desiredConditions is a map, so iteration order is random. Sort by type to keep the stored + // conditions stable across reconciles - otherwise every write reorders the list, which counts + // as a change and triggers another reconcile. + sort.Slice(statusConditions, func(i, j int) bool { + return statusConditions[i].Type < statusConditions[j].Type + }) return statusConditions } diff --git a/pkg/controller/logstorage/linseed/linseed_controller.go b/pkg/controller/logstorage/linseed/linseed_controller.go index 1c34680068..be405abd32 100644 --- a/pkg/controller/logstorage/linseed/linseed_controller.go +++ b/pkg/controller/logstorage/linseed/linseed_controller.go @@ -70,6 +70,7 @@ type LinseedSubController struct { multiTenant bool elasticExternal bool cloud bool + useSingleIndex bool } func Add(mgr manager.Manager, opts options.ControllerOptions) error { @@ -89,6 +90,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { status: status.New(mgr.GetClient(), "log-storage-access", opts.KubernetesVersion), elasticExternal: opts.ElasticExternal, cloud: opts.Cloud, + useSingleIndex: opts.UseSingleIndex, } r.status.Run(opts.ShutdownContext) @@ -336,7 +338,7 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(r.useSingleIndex) } // Determine the host and port from the URL. @@ -363,7 +365,8 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. // Query the username and password this Linseed instance should use to authenticate with Elasticsearch. // For multi-tenant systems, credentials are created by the elasticsearch users controller. - // For single-tenant system, these are created by es-kube-controllers. + // For single-tenant systems, these are created by es-kube-controllers, unless the cluster is + // migrating to single-index storage - then the users controller creates them as well. key = types.NamespacedName{Name: render.ElasticsearchLinseedUserSecret, Namespace: helper.InstallNamespace()} credentials := corev1.Secret{} if err = r.client.Get(ctx, key, &credentials); err != nil && !errors.IsNotFound(err) { @@ -420,7 +423,8 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. // Query the username and password this Linseed instance should use to authenticate with Elasticsearch. // For multi-tenant systems, credentials are created by the elasticsearch users controller. - // For single-tenant system, these are created by es-kube-controllers. + // For single-tenant systems, these are created by es-kube-controllers, unless the cluster is + // migrating to single-index storage - then the users controller creates them as well. // Delay installing Linseed until available. // TODO: Switch single-tenant to using operator-provisioned users. key = types.NamespacedName{Name: render.ElasticsearchLinseedUserSecret, Namespace: helper.InstallNamespace()} @@ -470,6 +474,7 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. ElasticClientCredentialsSecret: &credentials, LogStorage: logStorage, Cloud: r.cloud, + UseSingleIndex: r.useSingleIndex, } linseedComponent := linseed.Linseed(cfg) diff --git a/pkg/controller/logstorage/users/users_controller.go b/pkg/controller/logstorage/users/users_controller.go index c1deaf9e1f..e637f694b4 100644 --- a/pkg/controller/logstorage/users/users_controller.go +++ b/pkg/controller/logstorage/users/users_controller.go @@ -28,11 +28,13 @@ import ( "github.com/tigera/operator/pkg/render/logstorage/dashboards" corev1 "k8s.io/api/core/v1" + "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/crypto" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/common/secret" "k8s.io/apimachinery/pkg/api/errors" @@ -59,6 +61,11 @@ type UserController struct { esClientFn utils.ElasticsearchClientCreator multiTenant bool elasticExternal bool + + // useSingleIndex indicates that this single-tenant cluster stores its data in single-index format, + // in which case the users provisioned here are granted access to the single-index names rather than + // to the per-cluster multi-index ones. + useSingleIndex bool } type UsersCleanupController struct { @@ -72,9 +79,10 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if !opts.Variant.IsEnterprise() { return nil } - if !opts.MultiTenant { - // For now, the operator only creates users in multi-tenant mode. In single-tenant mode, - // user creation is handled by es-kube-controllers instead. + if !opts.MultiTenant && !opts.Cloud { + // The operator creates users for multi-tenant clusters, and for the single-tenant clusters of + // Calico Cloud - which build their tenant configuration from the cloud config ConfigMap. Anywhere + // else, user creation is handled by es-kube-controllers instead. return nil } @@ -83,6 +91,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { client: mgr.GetClient(), scheme: mgr.GetScheme(), multiTenant: opts.MultiTenant, + useSingleIndex: opts.UseSingleIndex, status: status.New(mgr.GetClient(), initializer.TigeraStatusLogStorageUsers, opts.KubernetesVersion), esClientFn: utils.NewElasticClient, elasticExternal: opts.ElasticExternal, @@ -119,6 +128,12 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err = c.WatchObject(&operatorv1.Tenant{}, &handler.EnqueueRequestForObject{}); err != nil { return fmt.Errorf("log-storage-user-controller failed to watch Tenant resource: %w", err) } + } else { + // In single-tenant mode, the tenant configuration - including the Elasticsearch endpoint - + // comes from the cloud config ConfigMap. + if err = utils.AddConfigMapWatch(c, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("log-storage-user-controller failed to watch the ConfigMap resource: %w", err) + } } // Watch for Elasticsearch. @@ -133,6 +148,11 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("log-storage-user-controller failed to create periodic reconcile watch: %w", err) } + if !opts.MultiTenant { + // The cleanup controller reconciles Tenant resources, which only exist in multi-tenant mode. + return nil + } + // Now that the users controller is set up, we can also set up the controller that cleans up stale users usersCleanupReconciler := &UsersCleanupController{ client: mgr.GetClient(), @@ -179,6 +199,23 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques return reconcile.Result{}, err } + if !r.multiTenant { + // Single-tenant clusters have no Tenant resource. Build the equivalent tenant configuration + // from the cloud config, so that we provision the users against the right Elasticsearch. + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if errors.IsNotFound(err) { + // The cloud config is written out of band, and may not exist yet. Wait for it rather than + // retrying with backoff - we watch the ConfigMap, so we reconcile again once it appears. + r.status.SetDegraded(operatorv1.ResourceNotReady, fmt.Sprintf("Waiting for ConfigMap %s to be available", cloudconfig.CloudConfigConfigMapName), nil, reqLogger) + return reconcile.Result{}, nil + } else if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) + return reconcile.Result{}, err + } + tenant = cloudConfig.ToTenant(r.useSingleIndex) + tenantID = tenant.Spec.ID + } + // Get LogStorage resource. logStorage := &operatorv1.LogStorage{} err = r.client.Get(ctx, utils.DefaultEnterpriseInstanceKey, logStorage) @@ -215,33 +252,25 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques } } - clusterIDConfigMap := corev1.ConfigMap{} - clusterIDConfigMapKey := client.ObjectKey{Name: "cluster-info", Namespace: "tigera-operator"} - err = r.client.Get(ctx, clusterIDConfigMapKey, &clusterIDConfigMap) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Waiting for ConfigMap %s/%s to be available", clusterIDConfigMapKey.Namespace, clusterIDConfigMapKey.Name), - nil, reqLogger) - return reconcile.Result{}, err - } - - clusterID, ok := clusterIDConfigMap.Data["cluster-id"] - if !ok { - err = fmt.Errorf("%s/%s ConfigMap does not contain expected 'cluster-id' key", - clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) - return reconcile.Result{}, err - } - - if clusterID == "" { - err = fmt.Errorf("%s/%s ConfigMap value for key 'cluster-id' must be non-empty", - clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) - return reconcile.Result{}, err + // Determine the names of the users to provision. In multi-tenant clusters the cluster ID forms part + // of the user names, and is read from the cluster-info ConfigMap written at install time. + // Single-tenant clusters have no such ConfigMap - their user names are derived from the tenant ID + // alone, matching the names es-kube-controllers used before the operator took over provisioning. + var linseedUser, dashboardUser *utils.User + if r.multiTenant { + clusterID, err := r.clusterID(ctx, reqLogger) + if err != nil { + return reconcile.Result{}, err + } + linseedUser = utils.LinseedUser(clusterID, tenant) + dashboardUser = utils.DashboardUser(clusterID, tenantID) + } else { + linseedUser = utils.LinseedUserSingleTenant(tenant, r.elasticExternal) + dashboardUser = utils.DashboardUserSingleTenant(tenantID) } // Query any existing username and password for this Linseed instance. If one already exists, we'll simply // use that. Otherwise, generate a new one. - linseedUser := utils.LinseedUser(clusterID, tenantID) linseedUserSecret := corev1.Secret{} var credentialSecrets []client.Object key := types.NamespacedName{Name: render.ElasticsearchLinseedUserSecret, Namespace: helper.TruthNamespace()} @@ -256,14 +285,19 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Make sure we install the generated credentials into the truth namespace. credentialSecrets = append(credentialSecrets, &linseedUserSecret) + } else if string(linseedUserSecret.Data["username"]) != linseedUser.Username { + // The credentials exist, but reference a different Elasticsearch user than the one we provision - + // e.g. because they were created by es-kube-controllers before the operator took over user + // provisioning. Point them at our user, keeping the existing password. + linseedUserSecret.StringData = map[string]string{"username": linseedUser.Username} + credentialSecrets = append(credentialSecrets, &linseedUserSecret) } // Query any existing username and password for this Dashboards instance. If one already exists, we'll simply // use that. Otherwise, generate a new one. keyDashboardCred := types.NamespacedName{Name: dashboards.ElasticCredentialsSecret, Namespace: helper.TruthNamespace()} - dashboardUser := utils.DashboardUser(clusterID, tenantID) dashboardUserSecret := corev1.Secret{} - if err = r.client.Get(ctx, key, &dashboardUserSecret); err != nil && !errors.IsNotFound(err) { + if err = r.client.Get(ctx, keyDashboardCred, &dashboardUserSecret); err != nil && !errors.IsNotFound(err) { r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error getting Secret %s", keyDashboardCred), err, reqLogger) return reconcile.Result{}, err } else if errors.IsNotFound(err) { @@ -274,6 +308,10 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Make sure we install the generated credentials into the truth namespace. credentialSecrets = append(credentialSecrets, &dashboardUserSecret) + } else if string(dashboardUserSecret.Data["username"]) != dashboardUser.Username { + // As above - point the existing credentials at the user we provision. + dashboardUserSecret.StringData = map[string]string{"username": dashboardUser.Username} + credentialSecrets = append(credentialSecrets, &dashboardUserSecret) } if helper.TruthNamespace() != helper.InstallNamespace() { @@ -297,7 +335,7 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques // Add a finalizer to the Tenant instance if it exists so that we can clean up the Linseed user when the Tenant // is deleted. The finalizer will be removed by the user cleanup controller when the user is deleted from ES. - if tenant != nil && tenant.GetDeletionTimestamp().IsZero() && !stringsutil.StringInSlice(userCleanupFinalizer, tenant.GetFinalizers()) { + if r.multiTenant && tenant != nil && tenant.GetDeletionTimestamp().IsZero() && !stringsutil.StringInSlice(userCleanupFinalizer, tenant.GetFinalizers()) { tenant.SetFinalizers(append(tenant.GetFinalizers(), userCleanupFinalizer)) if err = r.client.Update(ctx, tenant); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error adding finalizer to Tenant", err, reqLogger) @@ -325,6 +363,35 @@ func (r *UserController) Reconcile(ctx context.Context, request reconcile.Reques return reconcile.Result{}, nil } +// clusterID reads the cluster ID from the cluster-info ConfigMap. This ConfigMap is written at install +// time and only exists in multi-tenant clusters. +func (r *UserController) clusterID(ctx context.Context, reqLogger logr.Logger) (string, error) { + clusterIDConfigMap := corev1.ConfigMap{} + clusterIDConfigMapKey := client.ObjectKey{Name: "cluster-info", Namespace: "tigera-operator"} + if err := r.client.Get(ctx, clusterIDConfigMapKey, &clusterIDConfigMap); err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Waiting for ConfigMap %s/%s to be available", clusterIDConfigMapKey.Namespace, clusterIDConfigMapKey.Name), + nil, reqLogger) + return "", err + } + + clusterID, ok := clusterIDConfigMap.Data["cluster-id"] + if !ok { + err := fmt.Errorf("%s/%s ConfigMap does not contain expected 'cluster-id' key", + clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) + r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) + return "", err + } + + if clusterID == "" { + err := fmt.Errorf("%s/%s ConfigMap value for key 'cluster-id' must be non-empty", + clusterIDConfigMap.Namespace, clusterIDConfigMap.Name) + r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("%v", err), err, reqLogger) + return "", err + } + + return clusterID, nil +} + func (r *UserController) createUserLogin(ctx context.Context, elasticEndpoint string, secret *corev1.Secret, user *utils.User, reqLogger logr.Logger) error { esClient, err := r.esClientFn(r.client, ctx, elasticEndpoint, r.elasticExternal) if err != nil { @@ -419,7 +486,7 @@ func (r *UsersCleanupController) cleanupStaleUsers(ctx context.Context, logger l return fmt.Errorf("failed to fetch users from Elasticsearch") } - lu := utils.LinseedUser(clusterID, t.Spec.ID) + lu := utils.LinseedUser(clusterID, &t) dashboardsUser := utils.DashboardUser(clusterID, t.Spec.ID) for _, user := range allESUsers { if user.Username == lu.Username || user.Username == dashboardsUser.Username { diff --git a/pkg/controller/logstorage/users/users_controller_test.go b/pkg/controller/logstorage/users/users_controller_test.go index 1a8620c2f7..028f0424f9 100644 --- a/pkg/controller/logstorage/users/users_controller_test.go +++ b/pkg/controller/logstorage/users/users_controller_test.go @@ -23,15 +23,23 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" apiv1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" tigeraelastic "github.com/tigera/operator/pkg/controller/logstorage/elastic" + "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" + "github.com/tigera/operator/pkg/render/logstorage/dashboards" ) var _ = Describe("LogStorage cleanup controller", func() { @@ -61,17 +69,20 @@ var _ = Describe("LogStorage cleanup controller", func() { tenantID1 := "tenant1" tenantID2 := "tenant2" - staleLinseedUser := utils.LinseedUser(clusterID1, tenantID1) + tenant1 := &operatorv1.Tenant{Spec: operatorv1.TenantSpec{ID: tenantID1}} + tenant2 := &operatorv1.Tenant{Spec: operatorv1.TenantSpec{ID: tenantID2}} + + staleLinseedUser := utils.LinseedUser(clusterID1, tenant1) staleDashboardsUser := utils.DashboardUser(clusterID1, tenantID1) esTestUsers := []utils.User{ *staleLinseedUser, *staleDashboardsUser, - *utils.LinseedUser(clusterID1, tenantID2), + *utils.LinseedUser(clusterID1, tenant2), *utils.DashboardUser(clusterID1, tenantID2), - *utils.LinseedUser(clusterID2, tenantID1), + *utils.LinseedUser(clusterID2, tenant1), *utils.DashboardUser(clusterID2, tenantID1), - *utils.LinseedUser(clusterID2, tenantID2), + *utils.LinseedUser(clusterID2, tenant2), *utils.DashboardUser(clusterID2, tenantID2), } @@ -110,3 +121,142 @@ var _ = Describe("LogStorage cleanup controller", func() { Expect(testESClient.AssertExpectations(t)) }) }) + +// fakeESClient records the users provisioned against Elasticsearch. +type fakeESClient struct { + created []*utils.User +} + +func (f *fakeESClient) SetILMPolicies(context.Context, *operatorv1.LogStorage, bool) error { + return nil +} + +func (f *fakeESClient) CreateUser(_ context.Context, user *utils.User) error { + f.created = append(f.created, user) + return nil +} + +func (f *fakeESClient) DeleteUser(context.Context, *utils.User) error { return nil } + +func (f *fakeESClient) GetUsers(context.Context) ([]utils.User, error) { return nil, nil } + +var _ = Describe("LogStorage users controller", func() { + const ( + tenantID = "tenant-a" + ) + + var ( + cli client.Client + ctx context.Context + scheme *runtime.Scheme + esClient *fakeESClient + r *UserController + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(operatorv1.AddToScheme(scheme)).NotTo(HaveOccurred()) + Expect(corev1.AddToScheme(scheme)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + ctx = context.Background() + + ls := &operatorv1.LogStorage{} + ls.Name = "tigera-secure" + ls.Status.State = operatorv1.TigeraStatusReady + Expect(cli.Create(ctx, ls)).NotTo(HaveOccurred()) + + // Note that no cluster-info ConfigMap is created here: single-tenant clusters don't have one, + // and the controller must not require it. + Expect(cli.Create(ctx, cloudconfig.NewCloudConfig(tenantID, "tenant-a-name", "es.example.com", "kb.example.com", false).ConfigMap())).NotTo(HaveOccurred()) + + mockStatus := &status.MockStatus{} + mockStatus.On("OnCRFound").Return() + mockStatus.On("ReadyToMonitor") + mockStatus.On("ClearDegraded") + mockStatus.On("ClearWarning", mock.Anything).Return() + mockStatus.On("SetDegraded", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + + esClient = &fakeESClient{} + r = &UserController{ + client: cli, + scheme: scheme, + status: mockStatus, + esClientFn: func(_ client.Client, _ context.Context, _ string, _ bool) (utils.ElasticClient, error) { + return esClient, nil + }, + multiTenant: false, + elasticExternal: true, + useSingleIndex: true, + } + }) + + // singleTenant builds the Tenant that the controller derives from the cloud config ConfigMap created above. + singleTenant := func(useSingleIndex bool) *operatorv1.Tenant { + return cloudconfig.NewCloudConfig(tenantID, "tenant-a-name", "es.example.com", "kb.example.com", false).ToTenant(useSingleIndex) + } + + // secretValue reads a value from a Secret. The fake client doesn't convert StringData into Data + // the way the API server does, so we may find the value in either field. + secretValue := func(name, namespace, key string) string { + s := &corev1.Secret{} + ExpectWithOffset(1, cli.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, s)).NotTo(HaveOccurred()) + if v, ok := s.StringData[key]; ok { + return v + } + return string(s.Data[key]) + } + + It("should provision users for a single-tenant cluster using single-index storage", func() { + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + + // The Linseed user should be created in ES with the name es-kube-controllers used, and with + // privileges on the single-index calico_* indices. + expected := utils.LinseedUserSingleTenant(singleTenant(true), true) + Expect(expected.Username).To(Equal("tigera-ee-linseed-tenant-a-secure")) + Expect(esClient.created).To(HaveLen(2)) + Expect(esClient.created[0].Username).To(Equal(expected.Username)) + Expect(esClient.created[0].Roles).To(Equal(expected.Roles)) + Expect(esClient.created[0].Roles[0].Name).To(Equal(expected.Username)) + Expect(esClient.created[0].Roles[0].Definition.Indices[0].Names).To(ContainElement("calico_flowlogs_standard*")) + Expect(esClient.created[0].Roles[0].Definition.Indices[0].Names).To(HaveLen(len(operatorv1.DataTypes))) + Expect(esClient.created[1].Username).To(Equal("tigera-ee-dashboards-installer-tenant-a-secure")) + + // The credentials should be written to the Elasticsearch namespace for Linseed to consume. + Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "username")).To(Equal(expected.Username)) + Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "password")).NotTo(BeEmpty()) + Expect(secretValue(dashboards.ElasticCredentialsSecret, render.ElasticsearchNamespace, "username")).To(Equal(utils.DashboardUserSingleTenant(tenantID).Username)) + }) + + It("should re-point existing credentials at the operator provisioned user", func() { + // es-kube-controllers uses a different naming convention for the ES user. + Expect(cli.Create(ctx, &corev1.Secret{ + ObjectMeta: apiv1.ObjectMeta{Name: render.ElasticsearchLinseedUserSecret, Namespace: common.OperatorNamespace()}, + Data: map[string][]byte{"username": []byte("tigera-ee-linseed"), "password": []byte("existing-password")}, + })).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + + expected := utils.LinseedUserSingleTenant(singleTenant(true), true) + Expect(secretValue(render.ElasticsearchLinseedUserSecret, common.OperatorNamespace(), "username")).To(Equal(expected.Username)) + Expect(secretValue(render.ElasticsearchLinseedUserSecret, render.ElasticsearchNamespace, "username")).To(Equal(expected.Username)) + + // The existing password is preserved, and used for the user we provision. + Expect(secretValue(render.ElasticsearchLinseedUserSecret, common.OperatorNamespace(), "password")).To(Equal("existing-password")) + Expect(esClient.created[0].Username).To(Equal(expected.Username)) + Expect(esClient.created[0].Password).To(Equal("existing-password")) + }) + + It("should grant the Linseed user the multi-index names when not using single-index storage", func() { + r.useSingleIndex = false + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + + // The user keeps its name, but is only granted access to this tenant's multi-index format indices. + expected := utils.LinseedUserSingleTenant(singleTenant(false), true) + Expect(esClient.created[0].Username).To(Equal(expected.Username)) + Expect(esClient.created[0].Roles[0].Definition.Indices[0].Names).To(Equal([]string{"tigera_secure_ee_*.tenant-a.*.*"})) + }) +}) diff --git a/pkg/controller/manager/manager_controller_cloud.go b/pkg/controller/manager/manager_controller_cloud.go index 5f0c56591b..e9384b96b6 100644 --- a/pkg/controller/manager/manager_controller_cloud.go +++ b/pkg/controller/manager/manager_controller_cloud.go @@ -122,7 +122,7 @@ func (r *ReconcileManager) handleCloudReconcile( r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) return nil, render.ManagerCloudResources{}, nil, nil, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } else { var err error tenant, err = utils.GetTenantFromCloudAuthConfig(ctx, r.client) diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 7a7f0d8622..3bcf5276a7 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -61,6 +61,10 @@ type ControllerOptions struct { // LSS configuration and internal elasticsearch running. Only meaningful when Cloud is set. ESMigration bool + // UseSingleIndex is enabled in the last phase of an index migration for a single tenant cluster, + // during which the operator reconfigures log storage to use the single-index names. + UseSingleIndex bool + // Whether or not to use crd.projectcalico.org/v1 or projectcalico.org/v3 for Calico CRDs. UseV3CRDs bool diff --git a/pkg/controller/policyrecommendation/policyrecommendation_controller.go b/pkg/controller/policyrecommendation/policyrecommendation_controller.go index 3fdd36a259..b10c84d3f7 100644 --- a/pkg/controller/policyrecommendation/policyrecommendation_controller.go +++ b/pkg/controller/policyrecommendation/policyrecommendation_controller.go @@ -349,7 +349,7 @@ func (r *ReconcilePolicyRecommendation) Reconcile(ctx context.Context, request r r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, logc) return reconcile.Result{}, err } - tenant = cloudConfig.ToTenant() + tenant = cloudConfig.ToTenant(false) } // Create a component handler to manage the rendered component. diff --git a/pkg/controller/utils/elasticsearch.go b/pkg/controller/utils/elasticsearch.go index 0a977f291b..875d1857fe 100644 --- a/pkg/controller/utils/elasticsearch.go +++ b/pkg/controller/utils/elasticsearch.go @@ -220,8 +220,71 @@ var ( ElasticsearchUserNameDashboardInstaller = "tigera-ee-dashboards-installer" ) -func LinseedUser(clusterID, tenant string) *User { - username := formatName(ElasticsearchUserNameLinseed, clusterID, tenant) +// ElasticsearchSecureUserSuffix is appended to the user names provisioned for single-tenant clusters. +// It maintains the 1:1 mapping between the public user propagated to components and the private user +// swapped in at ES gateway, which strips this suffix. +const ElasticsearchSecureUserSuffix = "secure" + +// formatNameSingleTenant builds the ES user name for a single-tenant cluster: --secure. +// This matches the name previously provisioned by es-kube-controllers, so that existing credentials +// and role mappings keep resolving once the operator takes over user provisioning. +func formatNameSingleTenant(name, tenantID string) string { + return fmt.Sprintf("%s-%s-%s", name, tenantID, ElasticsearchSecureUserSuffix) +} + +// LinseedUser returns the Linseed user for a multi-tenant cluster. Multi-tenant clusters always store +// their data in single-index format, so the user is granted access to the indices declared on the Tenant. +func LinseedUser(clusterID string, tenant *operatorv1.Tenant) *User { + return linseedUser(formatName(ElasticsearchUserNameLinseed, clusterID, tenant.Spec.ID), singleIndexNames(tenant)) +} + +// LinseedUserSingleTenant returns the Linseed user for a single-tenant cluster. It is named the way +// es-kube-controllers named it, and needs no cluster ID. +// +// A single-tenant cluster only declares indices on its Tenant once it has moved to single-index storage; +// until then its data lives in the per-cluster multi-index format, and the user is granted access to +// that instead. +func LinseedUserSingleTenant(tenant *operatorv1.Tenant, externalElastic bool) *User { + names := multiIndexNames(tenant.Spec.ID, externalElastic) + if len(tenant.Spec.Indices) > 0 { + names = singleIndexNames(tenant) + } + return linseedUser(formatNameSingleTenant(ElasticsearchUserNameLinseed, tenant.Spec.ID), names) +} + +// singleIndexNames returns the index patterns covering the tenant's single-index storage. Each declared +// base index name is wildcarded so that the pattern matches the write alias as well as the numbered +// indices behind it. Indices without a base index name are skipped, so that a misconfigured Tenant +// cannot widen the pattern to "*" and grant access to every index in Elasticsearch. Tenants that +// declare no usable indices leave Linseed on its default index names, which are all prefixed with +// calico_. +func singleIndexNames(tenant *operatorv1.Tenant) []string { + names := make([]string, 0, len(tenant.Spec.Indices)) + for _, index := range tenant.Spec.Indices { + if index.BaseIndexName == "" { + continue + } + names = append(names, fmt.Sprintf("%s*", index.BaseIndexName)) + } + + if len(names) == 0 { + return []string{"calico_*"} + } + return names +} + +// multiIndexNames returns the index pattern covering multi-index storage, where every cluster writes to +// its own set of indices. Indices written to an Elasticsearch shared between tenants carry the tenant ID +// in their name, and the pattern is scoped to it. Indices written to the cluster's own Elasticsearch do +// not - Linseed has its tenant suffix disabled there - so there is nothing to scope the pattern to. +func multiIndexNames(tenant string, externalElastic bool) []string { + if !externalElastic { + tenant = "" + } + return []string{indexPattern("tigera_secure_ee_*", "*", ".*", tenant)} +} + +func linseedUser(username string, indices []string) *User { return &User{ Username: username, Roles: []Role{ @@ -231,8 +294,7 @@ func LinseedUser(clusterID, tenant string) *User { Cluster: []string{"monitor", "manage_index_templates", "manage_ilm"}, Indices: []RoleIndex{ { - // Include both single-index and multi-index name formats. - Names: []string{indexPattern("tigera_secure_ee_*", "*", ".*", tenant), "calico_*"}, + Names: indices, Privileges: []string{"create_index", "write", "manage", "read"}, }, }, @@ -243,7 +305,16 @@ func LinseedUser(clusterID, tenant string) *User { } func DashboardUser(clusterID, tenant string) *User { - username := formatName(ElasticsearchUserNameDashboardInstaller, clusterID, tenant) + return dashboardUser(formatName(ElasticsearchUserNameDashboardInstaller, clusterID, tenant)) +} + +// DashboardUserSingleTenant returns the Dashboards installer user for a single-tenant cluster, named +// the way es-kube-controllers named it. See LinseedUserSingleTenant. +func DashboardUserSingleTenant(tenant string) *User { + return dashboardUser(formatNameSingleTenant(ElasticsearchUserNameDashboardInstaller, tenant)) +} + +func dashboardUser(username string) *User { return &User{ Username: username, Roles: []Role{ diff --git a/pkg/controller/utils/utils_test.go b/pkg/controller/utils/utils_test.go index ecc9a33a1a..ffd83cddb4 100644 --- a/pkg/controller/utils/utils_test.go +++ b/pkg/controller/utils/utils_test.go @@ -291,7 +291,16 @@ var _ = Describe("Utils ElasticSearch test", func() { }) It("should generate Linseed ElasticUser with expected username and roles", func() { - linseedUser := LinseedUser(clusterID, tenantID) + tenant := &opv1.Tenant{ + Spec: opv1.TenantSpec{ + ID: tenantID, + Indices: []opv1.Index{ + {DataType: opv1.DataTypeFlowLogs, BaseIndexName: "calico_flowlogs_standard"}, + {DataType: opv1.DataTypeDNSLogs, BaseIndexName: "calico_dnslogs_standard"}, + }, + }, + } + linseedUser := LinseedUser(clusterID, tenant) expectedLinseedESName := fmt.Sprintf("%s_%s_%s", ElasticsearchUserNameLinseed, clusterID, tenantID) Expect(linseedUser.Username).To(Equal(expectedLinseedESName)) @@ -303,8 +312,8 @@ var _ = Describe("Utils ElasticSearch test", func() { Cluster: []string{"monitor", "manage_index_templates", "manage_ilm"}, Indices: []RoleIndex{ { - // Include both single-index and multi-index name formats. - Names: []string{indexPattern("tigera_secure_ee_*", "*", ".*", tenantID), "calico_*"}, + // The indices declared on the Tenant, wildcarded to cover the indices behind each alias. + Names: []string{"calico_flowlogs_standard*", "calico_dnslogs_standard*"}, Privileges: []string{"create_index", "write", "manage", "read"}, }, }, @@ -312,6 +321,56 @@ var _ = Describe("Utils ElasticSearch test", func() { Expect(*linseedRole.Definition).To(Equal(expectedLinseedRoleDef)) }) + + It("should grant a multi-tenant Linseed user the default single-index names when the Tenant declares none", func() { + tenant := &opv1.Tenant{Spec: opv1.TenantSpec{ID: tenantID}} + linseedUser := LinseedUser(clusterID, tenant) + + Expect(linseedUser.Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_*"})) + }) + + It("should generate a single-tenant Linseed user granted the indices its cluster stores data in", func() { + tenant := &opv1.Tenant{Spec: opv1.TenantSpec{ID: tenantID}} + expectedName := fmt.Sprintf("%s-%s-%s", ElasticsearchUserNameLinseed, tenantID, ElasticsearchSecureUserSuffix) + + // A cluster still on multi-index storage declares no indices, and gets access to the indices + // named for its tenant. + linseedUser := LinseedUserSingleTenant(tenant, true) + Expect(linseedUser.Username).To(Equal(expectedName)) + Expect(linseedUser.Roles[0].Name).To(Equal(expectedName)) + Expect(linseedUser.Roles[0].Definition.Indices[0].Names).To(Equal([]string{indexPattern("tigera_secure_ee_*", "*", ".*", tenantID)})) + + // Indices in the cluster's own Elasticsearch are not qualified by tenant, so neither is the pattern. + linseedUser = LinseedUserSingleTenant(tenant, false) + Expect(linseedUser.Roles[0].Definition.Indices[0].Names).To(Equal([]string{indexPattern("tigera_secure_ee_*", "*", ".*", "")})) + + // Once it moves to single-index storage, it gets access to the declared indices instead. + tenant.Spec.Indices = []opv1.Index{{DataType: opv1.DataTypeFlowLogs, BaseIndexName: "calico_flowlogs_standard"}} + linseedUser = LinseedUserSingleTenant(tenant, true) + Expect(linseedUser.Username).To(Equal(expectedName)) + Expect(linseedUser.Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_flowlogs_standard*"})) + }) + + It("should never widen the single-index pattern to all indices when a base index name is empty", func() { + // An index with no base index name must not be wildcarded into "*", which would grant Linseed + // access to every index in Elasticsearch. + tenant := &opv1.Tenant{ + Spec: opv1.TenantSpec{ + ID: tenantID, + Indices: []opv1.Index{ + {DataType: opv1.DataTypeFlowLogs, BaseIndexName: "calico_flowlogs_standard"}, + {DataType: opv1.DataTypeDNSLogs}, + }, + }, + } + Expect(LinseedUser(clusterID, tenant).Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_flowlogs_standard*"})) + Expect(LinseedUserSingleTenant(tenant, true).Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_flowlogs_standard*"})) + + // With no usable base index name at all, fall back to Linseed's default index names. + tenant.Spec.Indices = []opv1.Index{{DataType: opv1.DataTypeDNSLogs}} + Expect(LinseedUser(clusterID, tenant).Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_*"})) + Expect(LinseedUserSingleTenant(tenant, true).Roles[0].Definition.Indices[0].Names).To(Equal([]string{"calico_*"})) + }) }) type fakeClient struct { diff --git a/pkg/render/common/cloudconfig/cloudconfig.go b/pkg/render/common/cloudconfig/cloudconfig.go index f93dd3209e..a719194c97 100644 --- a/pkg/render/common/cloudconfig/cloudconfig.go +++ b/pkg/render/common/cloudconfig/cloudconfig.go @@ -16,6 +16,7 @@ package cloudconfig import ( "fmt" + "sort" "strconv" v1 "github.com/tigera/operator/api/v1" @@ -30,6 +31,25 @@ const ( CloudConfigConfigMapName = "tigera-secure-cloud-config" ) +// cloudStandardIndices maps each data type to the standard index base name used by clusters that +// have migrated to single-index storage. +var cloudStandardIndices = map[v1.DataType]string{ + v1.DataTypeAlerts: "calico_alerts_standard", + v1.DataTypeAuditLogs: "calico_auditlogs_standard", + v1.DataTypeBGPLogs: "calico_bgplogs_standard", + v1.DataTypeComplianceBenchmarks: "calico_compliance_benchmarks_results_standard", + v1.DataTypeComplianceReports: "calico_compliance_reports_standard", + v1.DataTypeComplianceSnapshots: "calico_compliance_snapshots_standard", + v1.DataTypeDNSLogs: "calico_dnslogs_standard", + v1.DataTypeFlowLogs: "calico_flowlogs_standard", + v1.DataTypeL7Logs: "calico_l7logs_standard", + v1.DataTypeRuntimeReports: "calico_runtime_reports_standard", + v1.DataTypeThreatFeedsDomainSet: "calico_threatfeeds_domainnameset_standard", + v1.DataTypeThreatFeedsIPSet: "calico_threatfeeds_ipset_standard", + v1.DataTypeWAFLogs: "calico_waflogs_standard", + v1.DataTypePolicyActivity: "calico_policy_activity_standard", +} + func NewCloudConfig(tenantId string, tenantName string, externalESDomain string, externalKibanaDomain string, enableMTLS bool) *CloudConfig { return &CloudConfig{ tenantId: tenantId, @@ -82,8 +102,12 @@ type CloudConfig struct { // ToTenant converts the given CloudConfig structure to a Tenant object. // This allows controllers that have been converted to support multi-tenancy to still leverage // the single-tenant CloudConfig structure using the same code path as in multi-tenancy. -func (c CloudConfig) ToTenant() *v1.Tenant { - return &v1.Tenant{ +// +// useSingleIndex declares the standard single-index base names on the returned Tenant. Only clusters +// migrating to single-index storage should set it: index base names are otherwise not carried on the +// artificial single-tenant Tenant, and components fall back to their default index names. +func (c CloudConfig) ToTenant(useSingleIndex bool) *v1.Tenant { + tenant := &v1.Tenant{ // We don't specify a Namespace for this tenant because it represents a singular tenant installed // in this management cluster. The signals to the render code that this is a single-tenant cluster and not // a cluster capable of multi-tenancy. @@ -98,6 +122,22 @@ func (c CloudConfig) ToTenant() *v1.Tenant { }, }, } + + if !useSingleIndex { + return tenant + } + + for dataType := range v1.DataTypes { + tenant.Spec.Indices = append(tenant.Spec.Indices, v1.Index{DataType: dataType, BaseIndexName: cloudStandardIndices[dataType]}) + } + + // DataTypes is a map, so iteration order is random. Sort by data type to keep the generated + // index list - and therefore the env vars rendered from it - stable across reconciles. + sort.Slice(tenant.Spec.Indices, func(i, j int) bool { + return tenant.Spec.Indices[i].DataType < tenant.Spec.Indices[j].DataType + }) + + return tenant } func (c CloudConfig) TenantId() string { diff --git a/pkg/render/common/cloudconfig/cloudconfig_test.go b/pkg/render/common/cloudconfig/cloudconfig_test.go index da125c5d73..b2e41d04ed 100644 --- a/pkg/render/common/cloudconfig/cloudconfig_test.go +++ b/pkg/render/common/cloudconfig/cloudconfig_test.go @@ -17,6 +17,7 @@ package cloudconfig import ( "strconv" + v1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" corev1 "k8s.io/api/core/v1" @@ -91,6 +92,52 @@ var _ = Describe("CloudConfig ConfigMap tests", func() { }) }) + Context("ToTenant", func() { + var cloudConfig *CloudConfig + + BeforeEach(func() { + cloudConfig = &CloudConfig{ + tenantId: "abc123", + tenantName: "tenant1", + externalESDomain: "externalES.com", + externalKibanaDomain: "externalKibana.com", + enableMTLS: true, + } + }) + + It("should return a single-tenant Tenant with the Elastic configuration from the CloudConfig", func() { + tenant := cloudConfig.ToTenant(false) + Expect(tenant.Name).To(Equal("default")) + Expect(tenant.Namespace).To(BeEmpty()) + Expect(tenant.MultiTenant()).To(BeFalse()) + Expect(tenant.Spec.ID).To(Equal("abc123")) + Expect(tenant.Spec.Name).To(Equal("tenant1")) + Expect(tenant.Spec.Elastic.URL).To(Equal("https://externalES.com:443")) + Expect(tenant.Spec.Elastic.KibanaURL).To(Equal("https://externalKibana.com:443")) + Expect(tenant.Spec.Elastic.MutualTLS).To(BeTrue()) + }) + + It("should not declare any indices when not using single-index storage", func() { + Expect(cloudConfig.ToTenant(false).Spec.Indices).To(BeEmpty()) + }) + + It("should declare the standard index for every data type when using single-index storage", func() { + indices := cloudConfig.ToTenant(true).Spec.Indices + Expect(indices).To(HaveLen(len(v1.DataTypes))) + for _, index := range indices { + Expect(index.BaseIndexName).To(Equal(cloudStandardIndices[index.DataType])) + Expect(index.BaseIndexName).ToNot(BeEmpty()) + } + }) + + It("should declare indices in a stable order", func() { + expected := cloudConfig.ToTenant(true).Spec.Indices + for i := 0; i < 10; i++ { + Expect(cloudConfig.ToTenant(true).Spec.Indices).To(Equal(expected)) + } + }) + }) + Context("ConfigMap from CloudConfig", func() { var cloudConfig *CloudConfig diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index f1c5e1f14e..980bd16db9 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -122,7 +122,9 @@ type KubeControllersConfiguration struct { TenantID string // Cloud indicates kube-controllers is being rendered for a Calico Cloud install. When false the - // cloud-specific RBAC below is not granted and enterprise RBAC is unchanged. + // cloud-specific RBAC below is not granted and enterprise RBAC is unchanged. In Calico Cloud, + // Elasticsearch configuration - including provisioning of the Linseed user - is handled by the + // operator's log-storage users controller rather than by es-kube-controllers. Cloud bool MetricsServerTLS certificatemanagement.KeyPairInterface @@ -282,7 +284,13 @@ func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeCon var enabledControllers []string if !cfg.Tenant.MultiTenant() { // Zero and single tenant cluster needs elasticsearch configuration - enabledControllers = append(enabledControllers, "authorization", "elasticsearchconfiguration") + enabledControllers = append(enabledControllers, "authorization") + if !cfg.Cloud { + // In Calico Cloud, the operator's log-storage users controller provisions the Elasticsearch + // users itself, so that they get the RBAC for the indices the cluster actually stores its + // data in. Running this controller as well would have it overwrite those users. + enabledControllers = append(enabledControllers, "elasticsearchconfiguration") + } if cfg.ManagementCluster != nil && cfg.Tenant == nil { // Enterprise will require the managedcluster controller to push licenses enabledControllers = append(enabledControllers, "managedcluster") diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 80db91bff7..8e82da5764 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -567,6 +567,24 @@ var _ = Describe("kube-controllers rendering tests", func() { })) }) + It("should not enable the elasticsearchconfiguration controller in Calico Cloud", func() { + instance.Variant = operatorv1.CalicoEnterprise + cfg.LogStorageExists = true + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + cfg.Cloud = true + + component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + envs := dp.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "authorization", + })) + }) + It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { expectedResources := []struct { name string diff --git a/pkg/render/logstorage/linseed/linseed.go b/pkg/render/logstorage/linseed/linseed.go index 66ea6522a9..6a2af9445c 100644 --- a/pkg/render/logstorage/linseed/linseed.go +++ b/pkg/render/logstorage/linseed/linseed.go @@ -117,6 +117,7 @@ type Config struct { // Tenant configuration, if running for a particular tenant. Tenant *operatorv1.Tenant ExternalElastic bool + UseSingleIndex bool // Secret containing client certificate and key for connecting to the Elastic cluster. If configured, // mTLS is used between Linseed and the external Elastic cluster. @@ -426,6 +427,13 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { if l.cfg.Tenant.Spec.ControlPlaneReplicas != nil { replicas = l.cfg.Tenant.Spec.ControlPlaneReplicas } + } else if l.cfg.UseSingleIndex { + // For single-tenant clusters migrating to single-index storage, + // use the elastic-single-index backend and configure index base names. + envVars = append(envVars, corev1.EnvVar{Name: "BACKEND", Value: "elastic-single-index"}) + for _, index := range l.cfg.Tenant.Spec.Indices { + envVars = append(envVars, index.EnvVar()) + } } } sc := securitycontext.NewNonRootContext()