diff --git a/cmd/member-net-controller-manager/main.go b/cmd/member-net-controller-manager/main.go index 3af57772..00f84de4 100644 --- a/cmd/member-net-controller-manager/main.go +++ b/cmd/member-net-controller-manager/main.go @@ -29,6 +29,7 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/cloud-provider-azure/pkg/azclient" "sigs.k8s.io/cloud-provider-azure/pkg/azclient/policy/ratelimit" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/privatelinkserviceclient" "sigs.k8s.io/cloud-provider-azure/pkg/azclient/publicipaddressclient" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -354,6 +355,12 @@ func setupControllersWithManager(ctx context.Context, hubMgr, memberMgr manager. } var azurePublicIPAddressClient publicipaddressclient.Interface + // azurePrivateLinkServicesClient is required for L7-FrontDoor exports; it + // is constructed alongside the PIP client whenever the Traffic Manager + // feature is enabled today, since both paths share the same cloud + // config and auth chain. Once the FrontDoor feature-gate lands + // (Phase 4) we may key this on its own toggle instead. + var azurePrivateLinkServicesClient privatelinkserviceclient.Interface var resourceGroupName string if *enableTrafficManagerFeature { klog.V(1).InfoS("Traffic manager feature is enabled, loading cloud config and creating azure clients", "cloudConfigFile", *cloudConfigFile) @@ -365,9 +372,9 @@ func setupControllersWithManager(ctx context.Context, hubMgr, memberMgr manager. cloudConfig.SetUserAgent("fleet-member-net-controller-manager") klog.V(1).InfoS("Cloud config loaded", "cloudConfig", cloudConfig) - azurePublicIPAddressClient, err = initAzureNetworkClients(cloudConfig) + azurePublicIPAddressClient, azurePrivateLinkServicesClient, err = initAzureNetworkClients(cloudConfig) if err != nil { - klog.ErrorS(err, "Unable to create Azure Traffic Manager clients") + klog.ErrorS(err, "Unable to create Azure network clients") return err } @@ -376,14 +383,15 @@ func setupControllersWithManager(ctx context.Context, hubMgr, memberMgr manager. klog.V(1).InfoS("Create serviceexport reconciler", "enableTrafficManagerFeature", *enableTrafficManagerFeature) if err := (&serviceexport.Reconciler{ - MemberClient: memberClient, - HubClient: hubClient, - MemberClusterID: mcName, - HubNamespace: mcHubNamespace, - Recorder: memberMgr.GetEventRecorderFor(serviceexport.ControllerName), - EnableTrafficManagerFeature: *enableTrafficManagerFeature, - ResourceGroupName: resourceGroupName, - AzurePublicIPAddressClient: azurePublicIPAddressClient, + MemberClient: memberClient, + HubClient: hubClient, + MemberClusterID: mcName, + HubNamespace: mcHubNamespace, + Recorder: memberMgr.GetEventRecorderFor(serviceexport.ControllerName), + EnableTrafficManagerFeature: *enableTrafficManagerFeature, + ResourceGroupName: resourceGroupName, + AzurePublicIPAddressClient: azurePublicIPAddressClient, + AzurePrivateLinkServicesClient: azurePrivateLinkServicesClient, }).SetupWithManager(memberMgr); err != nil { klog.ErrorS(err, "Unable to create serviceexport reconciler") return err @@ -404,11 +412,15 @@ func setupControllersWithManager(ctx context.Context, hubMgr, memberMgr manager. return nil } -// initAzureNetworkClients initializes the Azure network resource clients, currently only publicIPAddressClient. -func initAzureNetworkClients(cloudConfig *azure.CloudConfig) (publicipaddressclient.Interface, error) { +// initAzureNetworkClients initializes the Azure network resource clients used +// by the serviceexport reconciler: publicIPAddressClient (ATM / L4 path) and +// privateLinkServicesClient (Front Door / L7 path). Both share the same auth +// provider and rate-limit policy since they always run in the same +// subscription against the same cloud config. +func initAzureNetworkClients(cloudConfig *azure.CloudConfig) (publicipaddressclient.Interface, privatelinkserviceclient.Interface, error) { authProvider, err := azclient.NewAuthProvider(&cloudConfig.ARMClientConfig, &cloudConfig.AzureAuthConfig) if err != nil { - return nil, fmt.Errorf("failed to create Azure auth provider: %w", err) + return nil, nil, fmt.Errorf("failed to create Azure auth provider: %w", err) } factoryConfig := &azclient.ClientFactoryConfig{ @@ -417,7 +429,7 @@ func initAzureNetworkClients(cloudConfig *azure.CloudConfig) (publicipaddresscli } options, err := azclient.GetDefaultResourceClientOption(&cloudConfig.ARMClientConfig, factoryConfig) if err != nil { - return nil, fmt.Errorf("failed to get default resource client option: %w", err) + return nil, nil, fmt.Errorf("failed to get default resource client option: %w", err) } if rateLimitPolicy := ratelimit.NewRateLimitPolicy(cloudConfig.Config); rateLimitPolicy != nil { @@ -426,8 +438,13 @@ func initAzureNetworkClients(cloudConfig *azure.CloudConfig) (publicipaddresscli pipClient, err := publicipaddressclient.New(cloudConfig.SubscriptionID, authProvider.GetAzIdentity(), options) if err != nil { - return nil, fmt.Errorf("failed to create Azure PublicIPAddress client: %w", err) + return nil, nil, fmt.Errorf("failed to create Azure PublicIPAddress client: %w", err) } - return pipClient, nil + plsClient, err := privatelinkserviceclient.New(cloudConfig.SubscriptionID, authProvider.GetAzIdentity(), options) + if err != nil { + return nil, nil, fmt.Errorf("failed to create Azure PrivateLinkService client: %w", err) + } + + return pipClient, plsClient, nil } diff --git a/pkg/controllers/hub/frontdoorbackend/controller.go b/pkg/controllers/hub/frontdoorbackend/controller.go new file mode 100644 index 00000000..d940297c --- /dev/null +++ b/pkg/controllers/hub/frontdoorbackend/controller.go @@ -0,0 +1,651 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +// Package frontdoorbackend features the FrontDoorBackend controller (POC) +// that reconciles FrontDoorBackend CRs to Azure Front Door OriginGroups + +// Origins. +// +// Scope note (POC — mirrors the frontdoorprofile package's scope disclaimer): +// - Happy-path reconcile only. Detailed error classification, drift +// detection, and metrics are deferred past POC. +// - AFD/ATM coexistence guard (Conflict reason) lands in a follow-up +// commit; this file establishes the reconciler skeleton plus the +// Accepted / Invalid / Pending paths. +// +// Data model: +// - Each FrontDoorBackend produces ONE OriginGroup named fleet- +// under the parent FrontDoorProfile. +// - Each qualifying InternalServiceExport (ExportMode=L7-FrontDoor + +// PrivateLinkServiceResourceID populated) becomes ONE Origin under +// that OriginGroup, wired via SharedPrivateLinkResource to the +// exported PLS. +// - Per-origin weight = ceil(backend.Spec.Weight * export.Spec.Weight / +// sum(export.Spec.Weight)), matching TrafficManagerBackend semantics +// exactly so operators moving between the two surfaces get the same +// traffic distribution given the same inputs. +package frontdoorbackend + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cdn/armcdn/v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + fleetnetv1beta1 "go.goms.io/fleet-networking/api/v1beta1" + "go.goms.io/fleet-networking/pkg/common/azureerrors" + "go.goms.io/fleet-networking/pkg/common/objectmeta" + "go.goms.io/fleet-networking/pkg/controllers/hub/frontdoorprofile" +) + +const ( + // ControllerName is the name of the FrontDoorBackend controller. + ControllerName = "frontdoorbackend-controller" + + // AzureResourceOriginGroupNameFormat is the name of the Azure Front + // Door OriginGroup created for a FrontDoorBackend. Uses the CR UID so + // the underlying Azure name is stable across CR renames and + // deterministic per Kubernetes object — same convention as + // frontdoorprofile.AzureResourceProfileNameFormat. + AzureResourceOriginGroupNameFormat = "fleet-%s" + + // AzureResourceOriginNameFormat is the name of an Azure Front Door + // Origin under the OriginGroup: fleet--. The + // clusterID suffix keeps origins deterministic and lets us delete + // stale entries by prefix-scanning when a member cluster's export + // disappears. + AzureResourceOriginNameFormat = "fleet-%s-%s" + + // requeueOnPending is how often we requeue while waiting for the + // parent FrontDoorProfile to reach Programmed, or while a + // still-in-flight ServiceImport catches up with cluster status. Kept + // deliberately generous — SetupWithManager (Commit 10d) adds explicit + // secondary watches so most transitions are picked up without needing + // this fallback. + requeueOnPending = 30 * time.Second + + eventReasonAzureAPIError = "AzureAPIError" + eventReasonProgrammed = "Programmed" + eventReasonDeleted = "Deleted" + eventReasonInvalid = "Invalid" +) + +// Reconciler reconciles a FrontDoorBackend object. +type Reconciler struct { + client.Client + + // OriginGroupsClient CRUDs Microsoft.Cdn/profiles/*/originGroups. + OriginGroupsClient *armcdn.AFDOriginGroupsClient + // OriginsClient CRUDs Microsoft.Cdn/profiles/*/originGroups/*/origins. + OriginsClient *armcdn.AFDOriginsClient + + Recorder record.EventRecorder +} + +// AzureOriginGroupName returns the Azure OriginGroup name for a given +// FrontDoorBackend CR. Kept exported so the FrontDoorProfile deletion path +// (and any future FrontDoorRoute reconciler) can reference it without +// importing this file's private naming rules. +func AzureOriginGroupName(backend *fleetnetv1alpha1.FrontDoorBackend) string { + return fmt.Sprintf(AzureResourceOriginGroupNameFormat, backend.UID) +} + +// AzureOriginName returns the Azure Origin name for a given backend + member +// cluster. +func AzureOriginName(backend *fleetnetv1alpha1.FrontDoorBackend, clusterID string) string { + return fmt.Sprintf(AzureResourceOriginNameFormat, backend.UID, clusterID) +} + +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=frontdoorbackends,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=frontdoorbackends/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=frontdoorbackends/finalizers,verbs=get;update +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=frontdoorprofiles,verbs=get;list;watch +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=serviceimports,verbs=get;list;watch +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=internalserviceexports,verbs=get;list;watch +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=trafficmanagerbackends,verbs=get;list;watch +//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// Reconcile drives one reconciliation for a FrontDoorBackend. Mirrors the +// frontdoorprofile reconciler's structure so future readers can pattern-match +// between the two. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + name := req.NamespacedName + backendKRef := klog.KRef(name.Namespace, name.Name) + + startTime := time.Now() + klog.V(2).InfoS("Reconciliation starts", "frontDoorBackend", backendKRef) + defer func() { + klog.V(2).InfoS("Reconciliation ends", "frontDoorBackend", backendKRef, "latencyMs", time.Since(startTime).Milliseconds()) + }() + + backend := &fleetnetv1alpha1.FrontDoorBackend{} + if err := r.Client.Get(ctx, name, backend); err != nil { + if apierrors.IsNotFound(err) { + klog.V(2).InfoS("Ignoring NotFound frontDoorBackend", "frontDoorBackend", backendKRef) + return ctrl.Result{}, nil + } + klog.ErrorS(err, "Failed to get frontDoorBackend", "frontDoorBackend", backendKRef) + return ctrl.Result{}, err + } + + if !backend.ObjectMeta.DeletionTimestamp.IsZero() { + return r.handleDelete(ctx, backend) + } + return r.handleUpdate(ctx, backend) +} + +// handleDelete deletes the OriginGroup (which cascade-deletes its Origins) +// and removes the finalizer. NotFound from Azure is treated as success so +// the flow is idempotent across retries. +func (r *Reconciler) handleDelete(ctx context.Context, backend *fleetnetv1alpha1.FrontDoorBackend) (ctrl.Result, error) { + backendKObj := klog.KObj(backend) + if !controllerutil.ContainsFinalizer(backend, objectmeta.FrontDoorBackendFinalizer) { + return ctrl.Result{}, nil + } + + // We need the parent profile's resource group to address the + // OriginGroup. If the profile CR is already gone the AFD profile is + // likely gone too (OriginGroups cascade with it); we still attempt + // the delete + accept NotFound so a lingering FrontDoorBackend can + // always be finalized. + profile, err := r.getProfile(ctx, backend) + if err != nil && !apierrors.IsNotFound(err) { + klog.ErrorS(err, "Failed to look up parent FrontDoorProfile during delete", "frontDoorBackend", backendKObj) + return ctrl.Result{}, err + } + if profile != nil { + azProfileName := frontdoorprofile.AzureProfileName(profile) + azOriginGroupName := AzureOriginGroupName(backend) + poller, err := r.OriginGroupsClient.BeginDelete(ctx, profile.Spec.ResourceGroup, azProfileName, azOriginGroupName, nil) + if err != nil { + if !azureerrors.IsNotFound(err) { + return r.reportAzureError(ctx, backend, "begin delete origin group", err) + } + } else if _, err := poller.PollUntilDone(ctx, nil); err != nil { + if !azureerrors.IsNotFound(err) { + return r.reportAzureError(ctx, backend, "delete origin group", err) + } + } + r.Recorder.Eventf(backend, corev1.EventTypeNormal, eventReasonDeleted, + "AFD OriginGroup %s deleted", azOriginGroupName) + } + + controllerutil.RemoveFinalizer(backend, objectmeta.FrontDoorBackendFinalizer) + if err := r.Client.Update(ctx, backend); err != nil { + klog.ErrorS(err, "Failed to remove frontDoorBackend finalizer", "frontDoorBackend", backendKObj) + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +// handleUpdate is the reconcile path for a live FrontDoorBackend. +func (r *Reconciler) handleUpdate(ctx context.Context, backend *fleetnetv1alpha1.FrontDoorBackend) (ctrl.Result, error) { + backendKObj := klog.KObj(backend) + + // 1. Resolve parent FrontDoorProfile. Missing → Invalid (user error); + // not-yet-programmed → Pending. Both terminate this reconcile; + // SetupWithManager (Commit 10d) wires a Profile watch that + // re-triggers us when the profile flips Programmed=True. + profile, err := r.getProfile(ctx, backend) + if err != nil { + if apierrors.IsNotFound(err) { + return r.setInvalidAndUpdate(ctx, backend, fmt.Sprintf("FrontDoorProfile %q not found", backend.Spec.Profile.Name)) + } + return ctrl.Result{}, err + } + if !isProfileProgrammed(profile) { + return r.setPendingAndUpdate(ctx, backend, fmt.Sprintf("FrontDoorProfile %q is not yet programmed", profile.Name)) + } + + // 2. Resolve the ServiceImport. Missing → Invalid; no clusters → + // Pending (still waiting on member clusters to export). + svcImport := &fleetnetv1alpha1.ServiceImport{} + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: backend.Namespace, Name: backend.Spec.Backend.Name}, svcImport); err != nil { + if apierrors.IsNotFound(err) { + return r.setInvalidAndUpdate(ctx, backend, fmt.Sprintf("ServiceImport %q not found", backend.Spec.Backend.Name)) + } + return ctrl.Result{}, err + } + if len(svcImport.Status.Clusters) == 0 { + return r.setPendingAndUpdate(ctx, backend, "ServiceImport has no member cluster exports yet") + } + + // 2a. AFD/ATM coexistence guard. If a TrafficManagerBackend in the + // same namespace already claims this ServiceImport, refuse to + // program AFD origins for it — running both surfaces against the + // same backing service double-programs traffic and (worse) + // splits the L4 (public-IP) and L7 (Private-Link) exposure + // models, which SFI-NS253 explicitly forbids (see + // docs/first-party/001 §3.5). + // + // This is a terminal, user-visible error: Accepted=False, + // Reason=Conflict. The user has to explicitly delete the + // TrafficManagerBackend (or its FrontDoorBackend twin) to + // resolve; we don't guess a winner. When the loser is deleted, + // the TMB reconciler's list-based watch on our namespace won't + // wake us, so SetupWithManager (Commit 10d) adds an explicit + // TMB watch that enqueues same-namespace FrontDoorBackends. + if conflict, err := r.findConflictingTrafficManagerBackend(ctx, backend); err != nil { + return ctrl.Result{}, err + } else if conflict != nil { + return r.setConflictAndUpdate(ctx, backend, + fmt.Sprintf("TrafficManagerBackend %q already claims ServiceImport %q; delete one of the two backends to resolve", + conflict.Name, backend.Spec.Backend.Name)) + } + + // 3. List InternalServiceExports feeding this ServiceImport and keep + // only those that opted into the L7-FrontDoor path AND have a + // resolved PLS resource ID. Everything else is either the ATM + // surface's export or still waiting on cloud-provider-azure to + // finish provisioning its PLS. + exports := &fleetnetv1alpha1.InternalServiceExportList{} + if err := r.Client.List(ctx, exports, client.InNamespace(backend.Namespace)); err != nil { + return ctrl.Result{}, err + } + eligible := filterEligibleExports(exports.Items, svcImport) + if len(eligible) == 0 { + return r.setPendingAndUpdate(ctx, backend, + "No InternalServiceExport with ExportMode=L7-FrontDoor and a resolved PrivateLinkServiceResourceID yet") + } + + // 4. Register finalizer immediately before we contact Azure so a + // persistent 403 (bad RG, missing role assignment) cannot leave + // the CR undeletable — same rule the profile + TMB reconcilers + // follow. + if !controllerutil.ContainsFinalizer(backend, objectmeta.FrontDoorBackendFinalizer) { + controllerutil.AddFinalizer(backend, objectmeta.FrontDoorBackendFinalizer) + if err := r.Update(ctx, backend); err != nil { + klog.ErrorS(err, "Failed to add finalizer to frontDoorBackend", "frontDoorBackend", backendKObj) + return ctrl.Result{}, err + } + } + + // 5. Upsert the OriginGroup + one Origin per eligible export, then + // reflect what we programmed into .status. + azProfileName := frontdoorprofile.AzureProfileName(profile) + azOriginGroupName := AzureOriginGroupName(backend) + ogRes, err := r.ensureOriginGroup(ctx, profile.Spec.ResourceGroup, azProfileName, azOriginGroupName) + if err != nil { + return r.reportAzureError(ctx, backend, "ensure origin group", err) + } + + originStatuses, err := r.upsertOrigins(ctx, backend, profile.Spec.ResourceGroup, azProfileName, azOriginGroupName, eligible) + if err != nil { + return r.reportAzureError(ctx, backend, "upsert origins", err) + } + + backend.Status.OriginGroupResourceID = derefString(ogRes.ID) + backend.Status.Origins = originStatuses + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: string(fleetnetv1alpha1.FrontDoorBackendConditionAccepted), + Status: metav1.ConditionTrue, + ObservedGeneration: backend.Generation, + Reason: string(fleetnetv1alpha1.FrontDoorBackendReasonAccepted), + Message: fmt.Sprintf("Programmed %d origin(s) under OriginGroup %s", len(originStatuses), azOriginGroupName), + }) + if err := r.Client.Status().Update(ctx, backend); err != nil { + klog.ErrorS(err, "Failed to update frontDoorBackend status", "frontDoorBackend", backendKObj) + return ctrl.Result{}, err + } + r.Recorder.Eventf(backend, corev1.EventTypeNormal, eventReasonProgrammed, + "AFD OriginGroup %s programmed with %d origin(s)", azOriginGroupName, len(originStatuses)) + return ctrl.Result{}, nil +} + +// getProfile resolves the parent FrontDoorProfile in the same namespace as +// the backend. Returns apierrors.NotFound when the CR is missing so callers +// can distinguish "user error" from "transient API failure". +func (r *Reconciler) getProfile(ctx context.Context, backend *fleetnetv1alpha1.FrontDoorBackend) (*fleetnetv1alpha1.FrontDoorProfile, error) { + profile := &fleetnetv1alpha1.FrontDoorProfile{} + err := r.Client.Get(ctx, types.NamespacedName{Namespace: backend.Namespace, Name: backend.Spec.Profile.Name}, profile) + if err != nil { + return nil, err + } + return profile, nil +} + +// isProfileProgrammed returns true only when the profile carries a +// Programmed=True condition observing the current generation. We check +// ObservedGeneration explicitly so a stale True from before a profile spec +// change doesn't cause us to program origins against a half-migrated +// profile. +func isProfileProgrammed(profile *fleetnetv1alpha1.FrontDoorProfile) bool { + cond := meta.FindStatusCondition(profile.Status.Conditions, string(fleetnetv1alpha1.FrontDoorProfileConditionProgrammed)) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == profile.Generation +} + +// eligibleExport carries an export we are going to program together with +// its resolved raw weight. Kept as a private carrier struct so the +// per-origin weight math is easy to unit-test in isolation. +type eligibleExport struct { + Export *fleetnetv1alpha1.InternalServiceExport + // EffectiveWeight starts as the raw export weight (default 1 when + // absent) and is overwritten in-place by distributeWeights with the + // final per-origin weight. + EffectiveWeight int64 +} + +// filterEligibleExports keeps only exports that: +// - target this ServiceImport (matched via +// ServiceReference.Namespace+Name), AND +// - opted into the L7-FrontDoor mode, AND +// - already have a resolved PrivateLinkServiceResourceID. +// +// The ATM path's InternalServiceExports remain in the list but are dropped +// here. The AFD/ATM coexistence guard (Commit 10c) rejects the backend +// entirely when a competing TrafficManagerBackend already owns the same +// ServiceImport; that check will live in a separate helper for +// testability. +func filterEligibleExports(all []fleetnetv1alpha1.InternalServiceExport, svcImport *fleetnetv1alpha1.ServiceImport) []eligibleExport { + out := make([]eligibleExport, 0, len(all)) + for i := range all { + exp := &all[i] + if exp.Spec.ServiceReference.Namespace != svcImport.Namespace || + exp.Spec.ServiceReference.Name != svcImport.Name { + continue + } + if exp.Spec.ExportMode != objectmeta.ExportModeValueFrontDoor { + continue + } + if exp.Spec.PrivateLinkServiceResourceID == nil || *exp.Spec.PrivateLinkServiceResourceID == "" { + continue + } + // Default per-export weight to 1 to match TrafficManagerBackend + // behaviour when the annotation is absent on the source + // ServiceExport. + w := int64(1) + if exp.Spec.Weight != nil { + w = *exp.Spec.Weight + } + out = append(out, eligibleExport{Export: exp, EffectiveWeight: w}) + } + return out +} + +// distributeWeights rewrites EffectiveWeight in-place using the same +// ceil(backend.Weight * export.Weight / totalExportWeight) formula as +// TrafficManagerBackend, so the two surfaces produce identical traffic +// distributions given identical inputs. See TrafficManagerBackendSpec.Weight +// godoc for the derivation. +func distributeWeights(list []eligibleExport, aggregate int64) { + if len(list) == 0 || aggregate == 0 { + return + } + var total int64 + for _, e := range list { + total += e.EffectiveWeight + } + if total == 0 { + return + } + for i := range list { + w := math.Ceil(float64(aggregate*list[i].EffectiveWeight) / float64(total)) + list[i].EffectiveWeight = int64(w) + } +} + +// ensureOriginGroup creates the OriginGroup if missing; otherwise returns +// the existing resource. We do NOT currently patch OriginGroup properties +// (health probe, session affinity) — those Spec knobs are Phase 4 work. +func (r *Reconciler) ensureOriginGroup(ctx context.Context, rg, profileName, name string) (*armcdn.AFDOriginGroup, error) { + got, err := r.OriginGroupsClient.Get(ctx, rg, profileName, name, nil) + if err == nil { + return &got.AFDOriginGroup, nil + } + if !azureerrors.IsNotFound(err) { + return nil, err + } + desired := desiredAzureOriginGroup() + poller, err := r.OriginGroupsClient.BeginCreate(ctx, rg, profileName, name, desired, nil) + if err != nil { + return nil, err + } + res, err := poller.PollUntilDone(ctx, nil) + if err != nil { + return nil, err + } + return &res.AFDOriginGroup, nil +} + +// desiredAzureOriginGroup builds a minimal OriginGroup payload. The only +// hard requirement AFD imposes is a LoadBalancingSettings block; we use the +// service-default values (see Azure Front Door docs → "Origin group") since +// the CRD does not (yet) surface health-probe knobs. +func desiredAzureOriginGroup() armcdn.AFDOriginGroup { + return armcdn.AFDOriginGroup{ + Properties: &armcdn.AFDOriginGroupProperties{ + LoadBalancingSettings: &armcdn.LoadBalancingSettingsParameters{ + SampleSize: ptr.To[int32](4), + SuccessfulSamplesRequired: ptr.To[int32](3), + AdditionalLatencyInMilliseconds: ptr.To[int32](50), + }, + }, + } +} + +// upsertOrigins programs one Origin per eligible export under the origin +// group. We compute distributed weights first so a single mid-loop failure +// cannot produce a mismatch between what we programmed and what we +// reported. +func (r *Reconciler) upsertOrigins(ctx context.Context, backend *fleetnetv1alpha1.FrontDoorBackend, rg, profileName, ogName string, eligible []eligibleExport) ([]fleetnetv1alpha1.FrontDoorOriginStatus, error) { + // Distribute aggregate weight across exports up-front. + distributed := make([]eligibleExport, len(eligible)) + copy(distributed, eligible) + aggregate := int64(1) + if backend.Spec.Weight != nil { + aggregate = *backend.Spec.Weight + } + distributeWeights(distributed, aggregate) + + statuses := make([]fleetnetv1alpha1.FrontDoorOriginStatus, 0, len(distributed)) + for _, ee := range distributed { + exp := ee.Export + clusterID := exp.Spec.ServiceReference.ClusterID + originName := AzureOriginName(backend, clusterID) + desired := desiredAzureOrigin(*exp.Spec.PrivateLinkServiceResourceID, ee.EffectiveWeight) + poller, err := r.OriginsClient.BeginCreate(ctx, rg, profileName, ogName, originName, desired, nil) + if err != nil { + return nil, err + } + res, err := poller.PollUntilDone(ctx, nil) + if err != nil { + return nil, err + } + statuses = append(statuses, fleetnetv1alpha1.FrontDoorOriginStatus{ + Name: originName, + ResourceID: derefString(res.ID), + PrivateLinkServiceResourceID: exp.Spec.PrivateLinkServiceResourceID, + Weight: ptr.To(ee.EffectiveWeight), + From: &fleetnetv1alpha1.FromCluster{ + ClusterStatus: fleetnetv1alpha1.ClusterStatus{Cluster: clusterID}, + Weight: exp.Spec.Weight, + }, + }) + } + return statuses, nil +} + +// desiredAzureOrigin builds an Origin backed by the given PLS. HostName is +// required by the AFD API but is ignored at runtime when +// SharedPrivateLinkResource is present (traffic is tunnelled via PLS); we +// pass the PLS ID again as a placeholder so the payload validates without +// leaking a public hostname. +func desiredAzureOrigin(plsResourceID string, weight int64) armcdn.AFDOrigin { + return armcdn.AFDOrigin{ + Properties: &armcdn.AFDOriginProperties{ + HostName: ptr.To(plsResourceID), + Weight: ptr.To(int32(weight)), //nolint:gosec // G115: weight is CRD-validated to 0..1000 (FrontDoorBackendSpec.Weight), always fits int32. + SharedPrivateLinkResource: &armcdn.SharedPrivateLinkResourceProperties{ + PrivateLink: &armcdn.ResourceReference{ + ID: ptr.To(plsResourceID), + }, + RequestMessage: ptr.To("Fleet-networking Front Door backend"), + }, + EnabledState: ptr.To(armcdn.EnabledStateEnabled), + }, + } +} + +// findConflictingTrafficManagerBackend returns the first +// TrafficManagerBackend in the same namespace as `backend` whose +// Spec.Backend.Name points at the same ServiceImport this FrontDoorBackend +// wants to program. Returns (nil, nil) when nothing conflicts. +// +// The check is intentionally list-based (rather than field-indexed) — the +// AFD/ATM coexistence guard runs at most once per FrontDoorBackend +// reconcile, and namespaces are expected to have single-digit backend +// counts, so a linear scan is fine and does not need a new indexer. +func (r *Reconciler) findConflictingTrafficManagerBackend(ctx context.Context, backend *fleetnetv1alpha1.FrontDoorBackend) (*fleetnetv1beta1.TrafficManagerBackend, error) { + tmbList := &fleetnetv1beta1.TrafficManagerBackendList{} + if err := r.Client.List(ctx, tmbList, client.InNamespace(backend.Namespace)); err != nil { + return nil, err + } + for i := range tmbList.Items { + tmb := &tmbList.Items[i] + if !tmb.DeletionTimestamp.IsZero() { + // A TMB tearing itself down is not a conflict — the + // user has already signalled intent to migrate. + continue + } + if tmb.Spec.Backend.Name == backend.Spec.Backend.Name { + return tmb, nil + } + } + return nil, nil +} + +// setConflictAndUpdate writes Accepted=False,Reason=Conflict. Kept separate +// from setInvalidAndUpdate so operators (and dashboards) can distinguish +// "user pointed at a nonexistent thing" from "user asked for two +// mutually-exclusive surfaces at once". +func (r *Reconciler) setConflictAndUpdate(ctx context.Context, backend *fleetnetv1alpha1.FrontDoorBackend, msg string) (ctrl.Result, error) { + r.Recorder.Eventf(backend, corev1.EventTypeWarning, string(fleetnetv1alpha1.FrontDoorBackendReasonConflict), "%s", msg) + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: string(fleetnetv1alpha1.FrontDoorBackendConditionAccepted), + Status: metav1.ConditionFalse, + ObservedGeneration: backend.Generation, + Reason: string(fleetnetv1alpha1.FrontDoorBackendReasonConflict), + Message: msg, + }) + return ctrl.Result{}, r.Client.Status().Update(ctx, backend) +} + +// setInvalidAndUpdate writes Accepted=False,Reason=Invalid and stops +// reconciling. Used for terminal user errors (missing profile / import). +func (r *Reconciler) setInvalidAndUpdate(ctx context.Context, backend *fleetnetv1alpha1.FrontDoorBackend, msg string) (ctrl.Result, error) { + r.Recorder.Eventf(backend, corev1.EventTypeWarning, eventReasonInvalid, "%s", msg) + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: string(fleetnetv1alpha1.FrontDoorBackendConditionAccepted), + Status: metav1.ConditionFalse, + ObservedGeneration: backend.Generation, + Reason: string(fleetnetv1alpha1.FrontDoorBackendReasonInvalid), + Message: msg, + }) + return ctrl.Result{}, r.Client.Status().Update(ctx, backend) +} + +// setPendingAndUpdate writes Accepted=Unknown,Reason=Pending. Used while +// waiting for parent objects (Profile, ServiceImport, InternalServiceExport) +// to catch up; SetupWithManager (Commit 10d) wires the appropriate +// watches so we get re-triggered without needing an explicit requeue. +func (r *Reconciler) setPendingAndUpdate(ctx context.Context, backend *fleetnetv1alpha1.FrontDoorBackend, msg string) (ctrl.Result, error) { + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: string(fleetnetv1alpha1.FrontDoorBackendConditionAccepted), + Status: metav1.ConditionUnknown, + ObservedGeneration: backend.Generation, + Reason: string(fleetnetv1alpha1.FrontDoorBackendReasonPending), + Message: msg, + }) + return ctrl.Result{}, r.Client.Status().Update(ctx, backend) +} + +// reportAzureError writes a Pending or Invalid condition depending on the +// error class, emits an event, and requeues with backoff. Same shape as +// frontdoorprofile.reportAzureError. +func (r *Reconciler) reportAzureError(ctx context.Context, backend *fleetnetv1alpha1.FrontDoorBackend, op string, azErr error) (ctrl.Result, error) { + backendKObj := klog.KObj(backend) + klog.ErrorS(azErr, "Azure Front Door operation failed", + "frontDoorBackend", backendKObj, "operation", op) + r.Recorder.Eventf(backend, corev1.EventTypeWarning, eventReasonAzureAPIError, + "AFD %s failed: %v", op, azErr) + + status := metav1.ConditionUnknown + reason := fleetnetv1alpha1.FrontDoorBackendReasonPending + if azureerrors.IsClientError(azErr) && !azureerrors.IsThrottled(azErr) { + // 4xx that is not 429 → we won't fix this by retrying. + status = metav1.ConditionFalse + reason = fleetnetv1alpha1.FrontDoorBackendReasonInvalid + } + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: string(fleetnetv1alpha1.FrontDoorBackendConditionAccepted), + Status: status, + ObservedGeneration: backend.Generation, + Reason: string(reason), + Message: fmt.Sprintf("AFD %s: %v", op, azErr), + }) + if err := r.Client.Status().Update(ctx, backend); err != nil { + klog.ErrorS(err, "Failed to update frontDoorBackend status after Azure error", + "frontDoorBackend", backendKObj) + } + return ctrl.Result{RequeueAfter: requeueOnPending}, azErr +} + +func derefString(s *string) string { + if s == nil { + return "" + } + return *s +} + +// SetupWithManager wires the reconciler with secondary watches on: +// - FrontDoorProfile: when a profile flips Programmed=True (or its +// resource group changes) all same-namespace backends must re-run so +// Pending states resolve without waiting for requeueOnPending. +// - TrafficManagerBackend: when a conflicting TMB is deleted, the +// coexistence guard (handleUpdate step 2a) must clear immediately. +// - InternalServiceExport: when an export finishes its PLS +// provisioning and flips its PrivateLinkServiceResourceID, the +// backend needs to program the new origin. +// +// Enqueue helpers list same-namespace FrontDoorBackends because none of +// the secondary resources back-reference the backend directly. +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + enqueueSameNamespace := func(ctx context.Context, obj client.Object) []reconcile.Request { + list := &fleetnetv1alpha1.FrontDoorBackendList{} + if err := r.Client.List(ctx, list, client.InNamespace(obj.GetNamespace())); err != nil { + klog.ErrorS(err, "Failed to list frontDoorBackends when enqueueing from secondary event", + "namespace", obj.GetNamespace(), "kind", fmt.Sprintf("%T", obj)) + return nil + } + reqs := make([]reconcile.Request, 0, len(list.Items)) + for i := range list.Items { + b := &list.Items[i] + reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: b.Namespace, Name: b.Name}}) + } + return reqs + } + + return ctrl.NewControllerManagedBy(mgr). + For(&fleetnetv1alpha1.FrontDoorBackend{}). + Watches(&fleetnetv1alpha1.FrontDoorProfile{}, handler.EnqueueRequestsFromMapFunc(enqueueSameNamespace)). + Watches(&fleetnetv1beta1.TrafficManagerBackend{}, handler.EnqueueRequestsFromMapFunc(enqueueSameNamespace)). + Watches(&fleetnetv1alpha1.InternalServiceExport{}, handler.EnqueueRequestsFromMapFunc(enqueueSameNamespace)). + Complete(r) +} diff --git a/pkg/controllers/hub/frontdoorbackend/controller_integration_test.go b/pkg/controllers/hub/frontdoorbackend/controller_integration_test.go new file mode 100644 index 00000000..5d2dd24e --- /dev/null +++ b/pkg/controllers/hub/frontdoorbackend/controller_integration_test.go @@ -0,0 +1,445 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package frontdoorbackend + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + fleetnetv1beta1 "go.goms.io/fleet-networking/api/v1beta1" + "go.goms.io/fleet-networking/pkg/common/objectmeta" + "go.goms.io/fleet-networking/pkg/controllers/hub/frontdoorprofile" + "go.goms.io/fleet-networking/test/common/azurefrontdoor/fakeprovider" +) + +// Integration coverage for the FrontDoorBackend controller. Each Context is +// self-contained: creates its own FrontDoorProfile / FrontDoorBackend and +// (re)deletes them in AfterEach so specs are order-independent. + +const ( + eventuallyTimeout = 30 * time.Second + eventuallyInterval = 250 * time.Millisecond + + testClusterA = "cluster-a" + testClusterB = "cluster-b" + + testPLSResourceIDFormat = "/subscriptions/pls-sub/resourceGroups/pls-rg/providers/Microsoft.Network/privateLinkServices/%s" +) + +// markProfileProgrammed flips a FrontDoorProfile to Programmed=True so the +// FrontDoorBackend reconciler sees it as ready. In real life this comes from +// the FrontDoorProfile reconciler; here we do it directly to keep specs +// focused on the backend under test. +func markProfileProgrammed(profile *fleetnetv1alpha1.FrontDoorProfile) { + fresh := &fleetnetv1alpha1.FrontDoorProfile{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: profile.Namespace, Name: profile.Name}, fresh)).To(Succeed()) + meta.SetStatusCondition(&fresh.Status.Conditions, metav1.Condition{ + Type: string(fleetnetv1alpha1.FrontDoorProfileConditionProgrammed), + Status: metav1.ConditionTrue, + ObservedGeneration: fresh.Generation, + Reason: string(fleetnetv1alpha1.FrontDoorProfileReasonProgrammed), + Message: "programmed by test", + }) + Expect(k8sClient.Status().Update(ctx, fresh)).To(Succeed()) +} + +// newProfile creates a minimal FrontDoorProfile pointing at the fake's +// DefaultResourceGroupName (so the OriginGroup fake accepts our writes). +func newProfile(name string) *fleetnetv1alpha1.FrontDoorProfile { + return &fleetnetv1alpha1.FrontDoorProfile{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}, + Spec: fleetnetv1alpha1.FrontDoorProfileSpec{ + ResourceGroup: fakeprovider.DefaultResourceGroupName, + Sku: fleetnetv1alpha1.FrontDoorProfileSkuPremium, + }, + } +} + +func newServiceImport(name string) *fleetnetv1alpha1.ServiceImport { + return &fleetnetv1alpha1.ServiceImport{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}, + } +} + +func setServiceImportClusters(si *fleetnetv1alpha1.ServiceImport, clusters ...string) { + fresh := &fleetnetv1alpha1.ServiceImport{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: si.Namespace, Name: si.Name}, fresh)).To(Succeed()) + fresh.Status.Clusters = nil + for _, c := range clusters { + fresh.Status.Clusters = append(fresh.Status.Clusters, fleetnetv1alpha1.ClusterStatus{Cluster: c}) + } + Expect(k8sClient.Status().Update(ctx, fresh)).To(Succeed()) +} + +// newInternalServiceExport builds an InternalServiceExport in +// ExportMode=L7-FrontDoor with the given PLS resource ID; the reconciler's +// filterEligibleExports treats these as programmable origins. +func newInternalServiceExport(name, svcName, cluster string, weight int64, withPLS bool) *fleetnetv1alpha1.InternalServiceExport { + exp := &fleetnetv1alpha1.InternalServiceExport{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}, + Spec: fleetnetv1alpha1.InternalServiceExportSpec{ + // Ports is required by the CRD; content is irrelevant to + // the FrontDoorBackend reconciler which only looks at + // ExportMode + PrivateLinkServiceResourceID. + Ports: []fleetnetv1alpha1.ServicePort{{ + Protocol: corev1.ProtocolTCP, + Port: 80, + }}, + ServiceReference: fleetnetv1alpha1.ExportedObjectReference{ + ClusterID: cluster, + Kind: "Service", + Namespace: testNamespace, + Name: svcName, + ResourceVersion: "1", + Generation: 1, + UID: types.UID(fmt.Sprintf("uid-%s-%s", cluster, svcName)), + NamespacedName: fmt.Sprintf("%s/%s", testNamespace, svcName), + }, + Weight: ptr.To(weight), + ExportMode: objectmeta.ExportModeValueFrontDoor, + }, + } + if withPLS { + exp.Spec.PrivateLinkServiceResourceID = ptr.To(fmt.Sprintf(testPLSResourceIDFormat, cluster)) + } + return exp +} + +func acceptedCondition(nn types.NamespacedName) *metav1.Condition { + backend := &fleetnetv1alpha1.FrontDoorBackend{} + if err := k8sClient.Get(ctx, nn, backend); err != nil { + return nil + } + return meta.FindStatusCondition(backend.Status.Conditions, string(fleetnetv1alpha1.FrontDoorBackendConditionAccepted)) +} + +var _ = Describe("FrontDoorBackend Controller Integration", func() { + // Every spec constructs its own profile+backend+import; AfterEach + // unconditionally deletes them. Using unique names per Context keeps + // specs order-independent even if a previous AfterEach flakes. + + Context("Happy path — Programmed with two eligible exports", func() { + var ( + profileName = "afdp-happy" + backendName = "afdb-happy" + svcName = "svc-happy" + ) + + It("programs one OriginGroup and one Origin per eligible export", func() { + By("creating a Programmed FrontDoorProfile") + profile := newProfile(profileName) + Expect(k8sClient.Create(ctx, profile)).To(Succeed()) + markProfileProgrammed(profile) + + By("creating a ServiceImport with two member-cluster entries") + si := newServiceImport(svcName) + Expect(k8sClient.Create(ctx, si)).To(Succeed()) + setServiceImportClusters(si, testClusterA, testClusterB) + + By("creating two eligible InternalServiceExports") + expA := newInternalServiceExport("ise-a", svcName, testClusterA, 2, true) + expB := newInternalServiceExport("ise-b", svcName, testClusterB, 1, true) + Expect(k8sClient.Create(ctx, expA)).To(Succeed()) + Expect(k8sClient.Create(ctx, expB)).To(Succeed()) + + By("creating the FrontDoorBackend") + backend := &fleetnetv1alpha1.FrontDoorBackend{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: backendName}, + Spec: fleetnetv1alpha1.FrontDoorBackendSpec{ + Profile: fleetnetv1alpha1.FrontDoorProfileRef{Name: profileName}, + Backend: fleetnetv1alpha1.FrontDoorBackendRef{Name: svcName}, + Weight: ptr.To[int64](100), + }, + } + Expect(k8sClient.Create(ctx, backend)).To(Succeed()) + nn := types.NamespacedName{Namespace: testNamespace, Name: backendName} + + By("expecting Accepted=True with two origins programmed") + Eventually(func(g Gomega) { + cond := acceptedCondition(nn) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(string(fleetnetv1alpha1.FrontDoorBackendReasonAccepted))) + + // Fetch fresh copy so we see the reconciler's UID + // after Kubernetes assigned it. + fresh := &fleetnetv1alpha1.FrontDoorBackend{} + g.Expect(k8sClient.Get(ctx, nn, fresh)).To(Succeed()) + azProfile := frontdoorprofile.AzureProfileName(profile) + azOG := AzureOriginGroupName(fresh) + g.Expect(originGroupFake.Has(azProfile, azOG)).To(BeTrue()) + g.Expect(originFake.Count(azProfile, azOG)).To(Equal(2)) + g.Expect(fresh.Status.Origins).To(HaveLen(2)) + g.Expect(fresh.Status.OriginGroupResourceID).NotTo(BeEmpty()) + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + }) + + AfterEach(func() { + cleanupNamespaceObjects(profileName, backendName, svcName) + }) + }) + + Context("Pending — parent profile not yet Programmed", func() { + var ( + profileName = "afdp-pending" + backendName = "afdb-pending" + svcName = "svc-pending" + ) + + It("stays Accepted=Unknown/Pending until the profile flips", func() { + By("creating a FrontDoorProfile WITHOUT Programmed=True") + profile := newProfile(profileName) + Expect(k8sClient.Create(ctx, profile)).To(Succeed()) + + si := newServiceImport(svcName) + Expect(k8sClient.Create(ctx, si)).To(Succeed()) + setServiceImportClusters(si, testClusterA) + + exp := newInternalServiceExport("ise-p", svcName, testClusterA, 1, true) + Expect(k8sClient.Create(ctx, exp)).To(Succeed()) + + backend := &fleetnetv1alpha1.FrontDoorBackend{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: backendName}, + Spec: fleetnetv1alpha1.FrontDoorBackendSpec{ + Profile: fleetnetv1alpha1.FrontDoorProfileRef{Name: profileName}, + Backend: fleetnetv1alpha1.FrontDoorBackendRef{Name: svcName}, + Weight: ptr.To[int64](1), + }, + } + Expect(k8sClient.Create(ctx, backend)).To(Succeed()) + nn := types.NamespacedName{Namespace: testNamespace, Name: backendName} + + By("expecting Accepted=Unknown/Pending") + Eventually(func(g Gomega) { + cond := acceptedCondition(nn) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionUnknown)) + g.Expect(cond.Reason).To(Equal(string(fleetnetv1alpha1.FrontDoorBackendReasonPending))) + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + + By("flipping the profile to Programmed=True") + markProfileProgrammed(profile) + + By("expecting the backend to converge to Accepted=True (Profile watch wakes it)") + Eventually(func(g Gomega) { + cond := acceptedCondition(nn) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + }) + + AfterEach(func() { + cleanupNamespaceObjects(profileName, backendName, svcName) + }) + }) + + Context("Conflict — a TrafficManagerBackend already claims the same ServiceImport", func() { + var ( + profileName = "afdp-conflict" + backendName = "afdb-conflict" + svcName = "svc-conflict" + tmbName = "tmb-conflict" + ) + + It("refuses to program and sets Reason=Conflict, then clears when TMB is deleted", func() { + profile := newProfile(profileName) + Expect(k8sClient.Create(ctx, profile)).To(Succeed()) + markProfileProgrammed(profile) + + si := newServiceImport(svcName) + Expect(k8sClient.Create(ctx, si)).To(Succeed()) + setServiceImportClusters(si, testClusterA) + + exp := newInternalServiceExport("ise-c", svcName, testClusterA, 1, true) + Expect(k8sClient.Create(ctx, exp)).To(Succeed()) + + By("creating a TrafficManagerBackend that claims the same ServiceImport") + tmb := &fleetnetv1beta1.TrafficManagerBackend{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: tmbName}, + Spec: fleetnetv1beta1.TrafficManagerBackendSpec{ + Profile: fleetnetv1beta1.TrafficManagerProfileRef{Name: "atm-profile-not-used"}, + Backend: fleetnetv1beta1.TrafficManagerBackendRef{Name: svcName}, + Weight: ptr.To[int64](1), + }, + } + Expect(k8sClient.Create(ctx, tmb)).To(Succeed()) + + backend := &fleetnetv1alpha1.FrontDoorBackend{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: backendName}, + Spec: fleetnetv1alpha1.FrontDoorBackendSpec{ + Profile: fleetnetv1alpha1.FrontDoorProfileRef{Name: profileName}, + Backend: fleetnetv1alpha1.FrontDoorBackendRef{Name: svcName}, + Weight: ptr.To[int64](1), + }, + } + Expect(k8sClient.Create(ctx, backend)).To(Succeed()) + nn := types.NamespacedName{Namespace: testNamespace, Name: backendName} + + By("expecting Accepted=False/Reason=Conflict") + Eventually(func(g Gomega) { + cond := acceptedCondition(nn) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(string(fleetnetv1alpha1.FrontDoorBackendReasonConflict))) + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + + By("deleting the TMB") + Expect(k8sClient.Delete(ctx, tmb)).To(Succeed()) + + By("expecting the backend to converge to Accepted=True once the guard clears") + Eventually(func(g Gomega) { + cond := acceptedCondition(nn) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + }) + + AfterEach(func() { + cleanupNamespaceObjects(profileName, backendName, svcName) + // TMB may have already been deleted mid-spec; ignore NotFound. + tmb := &fleetnetv1beta1.TrafficManagerBackend{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: tmbName}, + } + _ = k8sClient.Delete(ctx, tmb) + }) + }) + + Context("Invalid — parent FrontDoorProfile does not exist", func() { + var ( + backendName = "afdb-invalid" + svcName = "svc-invalid" + ) + + It("sets Accepted=False/Reason=Invalid", func() { + backend := &fleetnetv1alpha1.FrontDoorBackend{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: backendName}, + Spec: fleetnetv1alpha1.FrontDoorBackendSpec{ + Profile: fleetnetv1alpha1.FrontDoorProfileRef{Name: "does-not-exist"}, + Backend: fleetnetv1alpha1.FrontDoorBackendRef{Name: svcName}, + Weight: ptr.To[int64](1), + }, + } + Expect(k8sClient.Create(ctx, backend)).To(Succeed()) + nn := types.NamespacedName{Namespace: testNamespace, Name: backendName} + + Eventually(func(g Gomega) { + cond := acceptedCondition(nn) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(string(fleetnetv1alpha1.FrontDoorBackendReasonInvalid))) + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + }) + + AfterEach(func() { + cleanupNamespaceObjects("", backendName, svcName) + }) + }) + + Context("Deletion — origin group is cleaned up", func() { + var ( + profileName = "afdp-del" + backendName = "afdb-del" + svcName = "svc-del" + ) + + It("deletes the OriginGroup and removes the finalizer", func() { + profile := newProfile(profileName) + Expect(k8sClient.Create(ctx, profile)).To(Succeed()) + markProfileProgrammed(profile) + + si := newServiceImport(svcName) + Expect(k8sClient.Create(ctx, si)).To(Succeed()) + setServiceImportClusters(si, testClusterA) + + exp := newInternalServiceExport("ise-d", svcName, testClusterA, 1, true) + Expect(k8sClient.Create(ctx, exp)).To(Succeed()) + + backend := &fleetnetv1alpha1.FrontDoorBackend{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: backendName}, + Spec: fleetnetv1alpha1.FrontDoorBackendSpec{ + Profile: fleetnetv1alpha1.FrontDoorProfileRef{Name: profileName}, + Backend: fleetnetv1alpha1.FrontDoorBackendRef{Name: svcName}, + Weight: ptr.To[int64](1), + }, + } + Expect(k8sClient.Create(ctx, backend)).To(Succeed()) + nn := types.NamespacedName{Namespace: testNamespace, Name: backendName} + + By("waiting until the backend has been programmed") + var azOG string + Eventually(func(g Gomega) { + cond := acceptedCondition(nn) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + fresh := &fleetnetv1alpha1.FrontDoorBackend{} + g.Expect(k8sClient.Get(ctx, nn, fresh)).To(Succeed()) + azOG = AzureOriginGroupName(fresh) + g.Expect(originGroupFake.Has(frontdoorprofile.AzureProfileName(profile), azOG)).To(BeTrue()) + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + + By("deleting the backend") + Expect(k8sClient.Delete(ctx, backend)).To(Succeed()) + + By("expecting the CR to disappear and the OriginGroup to be deleted from Azure") + Eventually(func(g Gomega) { + fresh := &fleetnetv1alpha1.FrontDoorBackend{} + err := k8sClient.Get(ctx, nn, fresh) + g.Expect(err).To(HaveOccurred()) + g.Expect(originGroupFake.Has(frontdoorprofile.AzureProfileName(profile), azOG)).To(BeFalse()) + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + }) + + AfterEach(func() { + cleanupNamespaceObjects(profileName, backendName, svcName) + }) + }) +}) + +// cleanupNamespaceObjects removes the profile/backend/serviceImport and all +// InternalServiceExports in the test namespace. Safe against NotFound so +// specs that half-created the fixture still clean up. +func cleanupNamespaceObjects(profileName, backendName, svcName string) { + if backendName != "" { + b := &fleetnetv1alpha1.FrontDoorBackend{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: backendName}, + } + _ = k8sClient.Delete(ctx, b) + // Wait for finalizer to drain so successive specs don't collide. + Eventually(func() bool { + return k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: backendName}, &fleetnetv1alpha1.FrontDoorBackend{}) != nil + }, eventuallyTimeout, eventuallyInterval).Should(BeTrue()) + } + if profileName != "" { + p := &fleetnetv1alpha1.FrontDoorProfile{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: profileName}, + } + _ = k8sClient.Delete(ctx, p) + } + if svcName != "" { + si := &fleetnetv1alpha1.ServiceImport{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: svcName}, + } + _ = k8sClient.Delete(ctx, si) + } + // Nuke any lingering InternalServiceExports in the namespace. + list := &fleetnetv1alpha1.InternalServiceExportList{} + if err := k8sClient.List(ctx, list); err == nil { + for i := range list.Items { + if list.Items[i].Namespace == testNamespace { + _ = k8sClient.Delete(ctx, &list.Items[i]) + } + } + } +} diff --git a/pkg/controllers/hub/frontdoorbackend/suite_test.go b/pkg/controllers/hub/frontdoorbackend/suite_test.go new file mode 100644 index 00000000..6ee4db09 --- /dev/null +++ b/pkg/controllers/hub/frontdoorbackend/suite_test.go @@ -0,0 +1,130 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package frontdoorbackend + +import ( + "context" + "flag" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + fleetnetv1beta1 "go.goms.io/fleet-networking/api/v1beta1" + "go.goms.io/fleet-networking/test/common/azurefrontdoor/fakeprovider" +) + +// Test-suite bootstrap for the FrontDoorBackend controller. Mirrors the +// frontdoorprofile suite so contributors reading either side see the same +// shape: envtest brings up etcd + kube-apiserver, controller-runtime wires +// the Reconciler with in-memory armcdn fakes, and Ginkgo specs run against +// the real cache/informer stack. + +var ( + cfg *rest.Config + mgr manager.Manager + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + originGroupFake *fakeprovider.OriginGroupFake + originFake *fakeprovider.OriginFake +) + +var testNamespace = fakeprovider.ProfileNamespace + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "FrontDoorBackend Controller Suite") +} + +var _ = BeforeSuite(func() { + logger := zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)) + klog.SetLogger(logger) + log.SetLogger(logger) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("../../../../", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + // Both API groups are needed: FrontDoorBackend + FrontDoorProfile + + // InternalServiceExport live in v1alpha1; TrafficManagerBackend + // (needed for the coexistence guard) is v1beta1. + Expect(fleetnetv1alpha1.AddToScheme(scheme.Scheme)).To(Succeed()) + Expect(fleetnetv1beta1.AddToScheme(scheme.Scheme)).To(Succeed()) + + By("constructing the k8s client") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + By("starting the controller manager") + klog.InitFlags(flag.CommandLine) + flag.Parse() + + mgr, err = ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + originGroupFake, err = fakeprovider.NewOriginGroupFake() + Expect(err).To(Succeed(), "failed to create fake AFD origin-groups client") + + originFake, err = fakeprovider.NewOriginFake() + Expect(err).To(Succeed(), "failed to create fake AFD origins client") + + Expect((&Reconciler{ + Client: mgr.GetClient(), + OriginGroupsClient: originGroupFake.Client, + OriginsClient: originFake.Client, + Recorder: mgr.GetEventRecorderFor(ControllerName), + }).SetupWithManager(mgr)).To(Succeed()) + + By("creating the test namespace") + ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}} + Expect(k8sClient.Create(ctx, &ns)).To(Succeed()) + + go func() { + defer GinkgoRecover() + Expect(mgr.Start(ctx)).To(Succeed(), "failed to run manager") + }() +}) + +var _ = AfterSuite(func() { + defer klog.Flush() + + By("deleting the test namespace") + ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}} + Expect(k8sClient.Delete(ctx, &ns)).To(Succeed()) + + cancel() + By("tearing down the test environment") + Expect(testEnv.Stop()).To(Succeed()) +}) diff --git a/pkg/controllers/member/serviceexport/controller.go b/pkg/controllers/member/serviceexport/controller.go index 359542d1..da7f3b93 100644 --- a/pkg/controllers/member/serviceexport/controller.go +++ b/pkg/controllers/member/serviceexport/controller.go @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "k8s.io/utils/ptr" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/privatelinkserviceclient" "sigs.k8s.io/cloud-provider-azure/pkg/azclient/publicipaddressclient" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -44,6 +45,23 @@ const ( svcExportInvalidIneligibleCondReason = "ServiceIneligible" svcExportPendingConflictResolutionReason = "ServicePendingConflictResolution" svcExportInvalidWeightAnnotationReason = "ServiceExportInvalidWeightAnnotation" + // svcExportInvalidExportModeReason is set when the export-mode annotation is + // syntactically invalid (typo, empty string, unsupported value). The + // export is halted until the operator corrects it. + svcExportInvalidExportModeReason = "ServiceExportInvalidExportModeAnnotation" + // svcExportPLSPendingReason is set while the Private Link Service is + // still being provisioned by cloud-provider-azure (ARM Get returns + // NotFound / nil). Mirrors the "public IP is in the progressing" + // branch — we return without requeuing and let the Service status + // update re-trigger reconciliation. + svcExportPLSPendingReason = "PrivateLinkServicePending" + + // annotationValueTrue is the case-sensitive "true" literal expected by + // cloud-provider-azure boolean annotations (e.g. azure-load-balancer-internal, + // azure-pls-create). Extracted to a const to satisfy goconst and to keep the + // three call-sites in sync with upstream's exact string comparison: + // https://github.com/kubernetes-sigs/cloud-provider-azure/blob/release-1.31/pkg/provider/azure_loadbalancer.go#L3559 + annotationValueTrue = "true" // svcExportCleanupFinalizer is the finalizer ServiceExport controllers adds to mark that // a ServiceExport can only be deleted after its corresponding Service has been unexported from the hub cluster. @@ -64,6 +82,11 @@ type Reconciler struct { ResourceGroupName string // default resource group name to create public IP address AzurePublicIPAddressClient publicipaddressclient.Interface + // AzurePrivateLinkServicesClient is used by the L7-FrontDoor export + // path to look up the Private Link Service that cloud-provider-azure + // provisioned for an internal LoadBalancer Service. Parallel to + // AzurePublicIPAddressClient (which serves the ATM path). + AzurePrivateLinkServicesClient privatelinkserviceclient.Interface EnableTrafficManagerFeature bool } @@ -196,6 +219,33 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, r.MemberClient.Status().Update(ctx, &svcExport) } + // Get the export mode from the annotation. Absent → default L4-TrafficManager + // (backward compatible). Invalid values are a hard reject: silently + // falling back to a default would let a Helm-typo like "L7-Frontdoor" + // mask user intent and route traffic through the wrong control plane. + exportMode, err := objectmeta.ExtractExportModeFromServiceExport(&svcExport) + if err != nil { + // Same pattern as invalid-weight: don't unexport (would drop + // live traffic on a good pre-existing export), don't requeue + // (annotation edits re-trigger reconcile), just surface the + // condition and stop. + klog.ErrorS(controller.NewUserError(err), "service export has invalid export-mode annotation", "service", svcRef) + curValidCond := meta.FindStatusCondition(svcExport.Status.Conditions, string(fleetnetv1beta1.ServiceExportValid)) + expectedValidCond := metav1.Condition{ + Type: string(fleetnetv1beta1.ServiceExportValid), + Status: metav1.ConditionFalse, + Reason: svcExportInvalidExportModeReason, + ObservedGeneration: svcExport.Generation, + Message: fmt.Sprintf("serviceExport %s/%s has an invalid export-mode annotation, err = %s", svcExport.Namespace, svcExport.Name, err), + } + if condition.EqualConditionWithMessage(curValidCond, &expectedValidCond) { + return ctrl.Result{}, nil + } + r.Recorder.Eventf(&svcExport, corev1.EventTypeWarning, svcExportInvalidExportModeReason, "ServiceExport %s has invalid export-mode value in the annotation", svc.Name) + meta.SetStatusCondition(&svcExport.Status.Conditions, expectedValidCond) + return ctrl.Result{}, r.MemberClient.Status().Update(ctx, &svcExport) + } + if exportWeight == 0 { // The weight is 0, unexport the service. klog.V(2).InfoS("Service has weight 0; unexport the service", "service", svcRef) @@ -250,11 +300,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } // Export the Service or update the exported Service. - return r.exportService(ctx, &svcExport, &svc, exportedSince, exportWeight) + return r.exportService(ctx, &svcExport, &svc, exportedSince, exportWeight, exportMode) } func (r *Reconciler) exportService(ctx context.Context, svcExport *fleetnetv1beta1.ServiceExport, svc *corev1.Service, - exportedSince time.Time, exportWeight int64) (ctrl.Result, error) { + exportedSince time.Time, exportWeight int64, exportMode string) (ctrl.Result, error) { svcRef := klog.KObj(svc) // Create or update the InternalServiceExport object. internalSvcExport := fleetnetv1alpha1.InternalServiceExport{ @@ -295,13 +345,37 @@ func (r *Reconciler) exportService(ctx context.Context, svcExport *fleetnetv1bet internalSvcExport.Spec.Ports = svcExportPorts internalSvcExport.Spec.ServiceReference.UpdateFromMetaObject(svc.ObjectMeta, metav1.NewTime(exportedSince)) - if r.EnableTrafficManagerFeature { - klog.V(2).InfoS("Collecting Traffic Manager related information and set to the internal service export", "service", svcRef) - internalSvcExport.Spec.Weight = ptr.To(exportWeight) - if err := r.setAzureRelatedInformation(ctx, svc, &internalSvcExport); err != nil { - klog.ErrorS(err, "Failed to populate the Azure information for the Traffic Manager feature in the internal service export", "service", svcRef) + // Propagate the resolved export mode to the hub. Hub reconcilers + // (TrafficManagerBackend for L4, FrontDoorBackend for L7) branch + // on this field to decide which control plane consumes the export. + // We deliberately leave the field empty for the L4-TrafficManager + // default so that (a) existing v1alpha1 objects retain their + // wire shape on upgrade and (b) the InternalServiceExport CRD + // contract "absent == L4-TrafficManager" is honoured on the wire, + // not just semantically. + if exportMode != objectmeta.ExportModeValueTrafficManager { + internalSvcExport.Spec.ExportMode = exportMode + } else { + internalSvcExport.Spec.ExportMode = "" + } + + switch exportMode { + case objectmeta.ExportModeValueFrontDoor: + klog.V(2).InfoS("Collecting Front Door / Private Link Service information for the internal service export", "service", svcRef) + if err := r.setAzureRelatedPrivateLinkInformation(ctx, svc, &internalSvcExport); err != nil { + klog.ErrorS(err, "Failed to populate the Private Link Service information for the Front Door feature in the internal service export", "service", svcRef) return err } + default: + // L4-TrafficManager (default) path — preserves today's behaviour. + if r.EnableTrafficManagerFeature { + klog.V(2).InfoS("Collecting Traffic Manager related information and set to the internal service export", "service", svcRef) + internalSvcExport.Spec.Weight = ptr.To(exportWeight) + if err := r.setAzureRelatedInformation(ctx, svc, &internalSvcExport); err != nil { + klog.ErrorS(err, "Failed to populate the Azure information for the Traffic Manager feature in the internal service export", "service", svcRef) + return err + } + } } return nil }) @@ -345,7 +419,7 @@ func (r *Reconciler) setAzureRelatedInformation(ctx context.Context, } // The annotation value is case-sensitive. // https://github.com/kubernetes-sigs/cloud-provider-azure/blob/release-1.31/pkg/provider/azure_loadbalancer.go#L3559 - hubSvcExport.Spec.IsInternalLoadBalancer = service.Annotations[objectmeta.ServiceAnnotationAzureLoadBalancerInternal] == "true" + hubSvcExport.Spec.IsInternalLoadBalancer = service.Annotations[objectmeta.ServiceAnnotationAzureLoadBalancerInternal] == annotationValueTrue if hubSvcExport.Spec.IsInternalLoadBalancer { // no need to populate the PublicIPResourceID and IsDNSLabelConfigured which are only applicable for external load balancer return nil @@ -402,8 +476,101 @@ func (r *Reconciler) setAzureRelatedInformation(ctx context.Context, return nil } -// TODO: can improve the performance by caching the public IP address resource ID. -// Note: we don't support "service.beta.kubernetes.io/azure-pip-prefix-id" annotation, and public ip cannot be found in +// setAzureRelatedPrivateLinkInformation populates the InternalServiceExport +// fields required by the L7-FrontDoor export path. Symmetric with +// setAzureRelatedInformation (ATM path) but resolves a Private Link Service +// (PLS) instead of a Public IP. +// +// Preconditions the underlying Service must satisfy (surfaced via +// ExportModeAnnotationServiceMismatch when violated): +// - Type == LoadBalancer AND is an internal LB. +// - Annotation "service.beta.kubernetes.io/azure-pls-create" == "true" +// so cloud-provider-azure actually provisions a PLS. +// - Annotation "service.beta.kubernetes.io/azure-pls-name" is set +// explicitly. We deliberately do not derive a default name from +// cloud-provider-azure internals: the derivation rules are private +// API surface and drift between upstream releases would silently +// point the reconciler at the wrong resource. +// +// PLS lookup uses the Get(rg, name) form (parallel with the PIP client) +// rather than List+filter — PLS has no ingress-IP style discriminator, so +// the annotation is the only stable identifier. +func (r *Reconciler) setAzureRelatedPrivateLinkInformation(ctx context.Context, + service *corev1.Service, + hubSvcExport *fleetnetv1alpha1.InternalServiceExport) error { + hubSvcExport.Spec.Type = service.Spec.Type + if service.Spec.Type != corev1.ServiceTypeLoadBalancer { + // FrontDoor path only makes sense for LoadBalancer Services; other + // types would have nowhere to attach a PLS. + return fmt.Errorf("L7-FrontDoor export requires Service type LoadBalancer, got %q", service.Spec.Type) + } + // The annotation value is case-sensitive; mirror the check used for + // the ATM path so a Service that is publicly-facing cannot be exported + // through the FrontDoor / PLS path by mistake. + hubSvcExport.Spec.IsInternalLoadBalancer = service.Annotations[objectmeta.ServiceAnnotationAzureLoadBalancerInternal] == annotationValueTrue + if !hubSvcExport.Spec.IsInternalLoadBalancer { + return fmt.Errorf("L7-FrontDoor export requires an internal LoadBalancer (annotation %q must be \"true\")", + objectmeta.ServiceAnnotationAzureLoadBalancerInternal) + } + + if service.Annotations[objectmeta.ServiceAnnotationAzurePLSCreate] != annotationValueTrue { + return fmt.Errorf("L7-FrontDoor export requires annotation %q to be \"true\"", + objectmeta.ServiceAnnotationAzurePLSCreate) + } + plsName := strings.TrimSpace(service.Annotations[objectmeta.ServiceAnnotationAzurePLSName]) + if plsName == "" { + return fmt.Errorf("L7-FrontDoor export requires an explicit PLS name via annotation %q", + objectmeta.ServiceAnnotationAzurePLSName) + } + // PLS RG resolution mirrors PIP RG resolution: dedicated PLS-RG + // annotation, then the shared LB-RG annotation, then the controller's + // default resource group. Keeping the fallback chain matched avoids + // operators having to duplicate annotations for the two paths. + rg := strings.TrimSpace(service.Annotations[objectmeta.ServiceAnnotationAzurePLSResourceGroup]) + if rg == "" { + rg = strings.TrimSpace(service.Annotations[objectmeta.ServiceAnnotationLoadBalancerResourceGroup]) + } + if rg == "" { + rg = r.ResourceGroupName + } + + serviceKObj := klog.KObj(service) + pls, err := r.AzurePrivateLinkServicesClient.Get(ctx, rg, plsName, nil) + if err != nil { + // Treat NotFound as "still provisioning" (mirrors the PIP nil + // branch below). cloud-provider-azure may not have created the + // PLS yet even though the Service exists; a Service status + // update or the annotation edit will re-trigger reconcile. + if isAzureNotFoundError(err) { + klog.V(2).InfoS("The Private Link Service is not created yet", "service", serviceKObj, "resourceGroup", rg, "name", plsName, "reason", svcExportPLSPendingReason) + return nil + } + klog.ErrorS(err, "Failed to get Azure Private Link Service", "service", serviceKObj, "resourceGroup", rg, "name", plsName) + return err + } + if pls == nil || pls.ID == nil { + // Defensive: some ARM SDK error paths surface (nil, nil). + klog.V(2).InfoS("The Private Link Service resource ID is not populated yet", "service", serviceKObj, "resourceGroup", rg, "name", plsName, "reason", svcExportPLSPendingReason) + return nil + } + hubSvcExport.Spec.PrivateLinkServiceResourceID = pls.ID + return nil +} + +// isAzureNotFoundError returns true when the error corresponds to an ARM +// 404 response. Kept as a small local helper (rather than pulling in +// azcore.ResponseError) to minimise imports; upgrade to azcore matching +// when we add richer error handling in Phase 4. +func isAzureNotFoundError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "ResourceNotFound") || + strings.Contains(msg, "NotFound") || + strings.Contains(msg, "StatusCode=404") +} + // this case. func (r *Reconciler) lookupPublicIPResourceIDByLoadBalancerIP(ctx context.Context, service *corev1.Service) (*armnetwork.PublicIPAddress, error) { // The customer can specify the resource group for the public IP address in the service annotation. diff --git a/pkg/controllers/member/serviceexport/controller_integration_test.go b/pkg/controllers/member/serviceexport/controller_integration_test.go index cba8e47e..1f00f568 100644 --- a/pkg/controllers/member/serviceexport/controller_integration_test.go +++ b/pkg/controllers/member/serviceexport/controller_integration_test.go @@ -43,6 +43,14 @@ const ( testIngressIP = "1.2.3.4" testPublicIPResourceID = "/subscriptions/sub1/resourceGroups/valid-rg/providers/Microsoft.Network/publicIPAddresses/pip" + + // L7-FrontDoor path fixtures. testPrivateLinkServiceName mirrors the + // value the integration spec sets on the Service's + // "service.beta.kubernetes.io/azure-pls-name" annotation; the fake PLS + // client in suite_test.go is seeded with a matching entry so the + // happy-path lookup returns testPrivateLinkServiceResourceID. + testPrivateLinkServiceName = "pls1" + testPrivateLinkServiceResourceID = "/subscriptions/sub1/resourceGroups/valid-rg/providers/Microsoft.Network/privateLinkServices/pls1" ) // clusterIPService returns a Service of ClusterIP type. @@ -113,6 +121,47 @@ func publicLoadBalancerService() *corev1.Service { } } +// internalLoadBalancerServiceWithPLS returns an internal LoadBalancer Service +// annotated so that cloud-provider-azure would provision a Private Link +// Service — the shape the L7-FrontDoor export path requires. The suite fake +// PLS client is seeded to match ServiceAnnotationAzurePLSName. +func internalLoadBalancerServiceWithPLS() *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: memberUserNS, + Name: svcName, + Annotations: map[string]string{ + objectmeta.ServiceAnnotationAzureLoadBalancerInternal: "true", + objectmeta.ServiceAnnotationAzurePLSCreate: "true", + objectmeta.ServiceAnnotationAzurePLSName: testPrivateLinkServiceName, + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Ports: []corev1.ServicePort{ + { + Port: svcPort, + TargetPort: intstr.FromInt32(targetPort), + }, + }, + }, + } +} + +// frontDoorServiceExport returns a ServiceExport annotated to opt into the +// L7-FrontDoor export path. +func frontDoorServiceExport() *fleetnetv1beta1.ServiceExport { + return &fleetnetv1beta1.ServiceExport{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: memberUserNS, + Name: svcName, + Annotations: map[string]string{ + objectmeta.ServiceExportAnnotationExportMode: objectmeta.ExportModeValueFrontDoor, + }, + }, + } +} + func notYetFulfilledServiceExport() *fleetnetv1beta1.ServiceExport { return &fleetnetv1beta1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ @@ -977,4 +1026,64 @@ var _ = Describe("serviceexport controller", func() { Eventually(serviceIsExportedToHubActual(svc.Spec.Type, true, ptr.To(int64(1))), eventuallyTimeout, eventuallyInterval).Should(Succeed()) }) }) + + // L7-FrontDoor export path: the ServiceExport is annotated with + // export-mode=L7-FrontDoor and the Service opts into PLS provisioning + // via the standard cloud-provider-azure annotations. The reconciler + // should propagate ExportMode + PrivateLinkServiceResourceID (resolved + // via the fake PLS client) onto the hub InternalServiceExport, and + // crucially must NOT populate any of the ATM-only fields. + Context("export internal load balancer service via L7-FrontDoor mode", func() { + var svc *corev1.Service + var svcExport *fleetnetv1beta1.ServiceExport + + BeforeEach(func() { + svc = internalLoadBalancerServiceWithPLS() + Expect(memberClient.Create(ctx, svc)).Should(Succeed()) + + svcExport = frontDoorServiceExport() + Expect(memberClient.Create(ctx, svcExport)).Should(Succeed()) + }) + + AfterEach(func() { + Expect(memberClient.Delete(ctx, svcExport)).Should(Succeed()) + Expect(memberClient.Delete(ctx, svc)).Should(Succeed()) + + Eventually(serviceIsNotExportedActual, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + Eventually(serviceExportIsAbsentActual, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + Eventually(serviceIsAbsentActual, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + }) + + It("should mark the service export as valid + should populate ExportMode and PrivateLinkServiceResourceID", func() { + Eventually(serviceIsExportedFromMemberActual, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + + Eventually(func() error { + internalSvcExport := &fleetnetv1alpha1.InternalServiceExport{} + if err := hubClient.Get(ctx, internalSvcExportKey, internalSvcExport); err != nil { + return fmt.Errorf("internalServiceExport Get(%+v), got %w, want no error", internalSvcExportKey, err) + } + if internalSvcExport.Spec.ExportMode != objectmeta.ExportModeValueFrontDoor { + return fmt.Errorf("internalServiceExport.Spec.ExportMode = %q, want %q", + internalSvcExport.Spec.ExportMode, objectmeta.ExportModeValueFrontDoor) + } + if internalSvcExport.Spec.PrivateLinkServiceResourceID == nil || + *internalSvcExport.Spec.PrivateLinkServiceResourceID != testPrivateLinkServiceResourceID { + return fmt.Errorf("internalServiceExport.Spec.PrivateLinkServiceResourceID = %v, want %q", + internalSvcExport.Spec.PrivateLinkServiceResourceID, testPrivateLinkServiceResourceID) + } + // L7 path must not touch the ATM-only fields — leaving these + // populated would confuse the hub TrafficManagerBackend + // reconciler, which today keys off Weight/PublicIPResourceID. + if internalSvcExport.Spec.PublicIPResourceID != nil { + return fmt.Errorf("internalServiceExport.Spec.PublicIPResourceID = %v, want nil (L7 path)", + internalSvcExport.Spec.PublicIPResourceID) + } + if internalSvcExport.Spec.Weight != nil { + return fmt.Errorf("internalServiceExport.Spec.Weight = %v, want nil (L7 path)", + internalSvcExport.Spec.Weight) + } + return nil + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + }) + }) }) diff --git a/pkg/controllers/member/serviceexport/controller_test.go b/pkg/controllers/member/serviceexport/controller_test.go index 1c76cbfa..dac5b11d 100644 --- a/pkg/controllers/member/serviceexport/controller_test.go +++ b/pkg/controllers/member/serviceexport/controller_test.go @@ -1603,3 +1603,211 @@ func (c *fakePublicIPAddressClient) List(_ context.Context, rg string) ([]*armne } return nil, errors.New("invalid resource group") } + +// fakePrivateLinkServicesClient implements +// sigs.k8s.io/cloud-provider-azure/pkg/azclient/privatelinkserviceclient.Interface +// with in-memory Get responses keyed by "/". Missing keys yield a +// synthetic NotFound error so the reconciler's still-provisioning branch is +// exercised without needing a real ARM error type. +type fakePrivateLinkServicesClient struct { + // GetResponses maps "/" -> PrivateLinkService returned by Get. + GetResponses map[string]*armnetwork.PrivateLinkService + // GetErrOverride, if set, is returned from every Get in place of the + // key lookup. Use for testing non-404 error paths. + GetErrOverride error +} + +func (c *fakePrivateLinkServicesClient) Get(_ context.Context, rg string, name string, _ *string) (*armnetwork.PrivateLinkService, error) { + if c.GetErrOverride != nil { + return nil, c.GetErrOverride + } + if pls, ok := c.GetResponses[rg+"/"+name]; ok { + return pls, nil + } + // Synthesise a NotFound error that contains the string the reconciler's + // isAzureNotFoundError helper looks for. Keeps the fake dependency-free. + return nil, errors.New("ResourceNotFound: no PLS at " + rg + "/" + name) +} + +func (c *fakePrivateLinkServicesClient) CreateOrUpdate(_ context.Context, _ string, _ string, _ armnetwork.PrivateLinkService) (*armnetwork.PrivateLinkService, error) { + return nil, nil +} + +func (c *fakePrivateLinkServicesClient) Delete(_ context.Context, _ string, _ string) error { + return nil +} + +func (c *fakePrivateLinkServicesClient) List(_ context.Context, _ string) ([]*armnetwork.PrivateLinkService, error) { + return nil, nil +} + +func TestSetAzureRelatedPrivateLinkInformation(t *testing.T) { + const ( + validPLSName = "pls1" + altResourceGroup = "custom-pls-rg" + validPLSResourceID = "/subscriptions/sub1/resourceGroups/valid-rg/providers/Microsoft.Network/privateLinkServices/pls1" + altPLSResourceID = "/subscriptions/sub1/resourceGroups/custom-pls-rg/providers/Microsoft.Network/privateLinkServices/pls1" + ) + tests := []struct { + name string + service *corev1.Service + getResponses map[string]*armnetwork.PrivateLinkService + getErrOverride error + wantPLSID *string + wantErr bool + }{ + { + name: "not a LoadBalancer service — hard reject", + service: &corev1.Service{ + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP}, + }, + wantErr: true, + }, + { + name: "LoadBalancer but not internal — hard reject", + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + // Note: only "true" (lowercase) qualifies; leaving it unset is + // treated as public LB, which is incompatible with L7-FrontDoor. + }, + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, + wantErr: true, + }, + { + name: "internal LB missing pls-create=true — hard reject", + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + objectmeta.ServiceAnnotationAzureLoadBalancerInternal: "true", + }, + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, + wantErr: true, + }, + { + name: "pls-create=true but pls-name missing — hard reject", + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + objectmeta.ServiceAnnotationAzureLoadBalancerInternal: "true", + objectmeta.ServiceAnnotationAzurePLSCreate: "true", + }, + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, + wantErr: true, + }, + { + name: "PLS not yet provisioned — soft pending, no error, no ID set", + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + objectmeta.ServiceAnnotationAzureLoadBalancerInternal: "true", + objectmeta.ServiceAnnotationAzurePLSCreate: "true", + objectmeta.ServiceAnnotationAzurePLSName: validPLSName, + }, + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, + // No entry in getResponses → fake returns a NotFound error → + // reconciler treats as still-provisioning and returns nil. + getResponses: map[string]*armnetwork.PrivateLinkService{}, + wantPLSID: nil, + wantErr: false, + }, + { + name: "PLS Get returns a non-404 error — bubble it up", + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + objectmeta.ServiceAnnotationAzureLoadBalancerInternal: "true", + objectmeta.ServiceAnnotationAzurePLSCreate: "true", + objectmeta.ServiceAnnotationAzurePLSName: validPLSName, + }, + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, + getErrOverride: errors.New("boom: throttled"), + wantErr: true, + }, + { + name: "happy path — PLS exists in default RG", + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + objectmeta.ServiceAnnotationAzureLoadBalancerInternal: "true", + objectmeta.ServiceAnnotationAzurePLSCreate: "true", + objectmeta.ServiceAnnotationAzurePLSName: validPLSName, + }, + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, + getResponses: map[string]*armnetwork.PrivateLinkService{ + validResourceGroup + "/" + validPLSName: {ID: ptr.To(validPLSResourceID)}, + }, + wantPLSID: ptr.To(validPLSResourceID), + }, + { + name: "happy path — pls-resource-group annotation overrides default RG", + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + objectmeta.ServiceAnnotationAzureLoadBalancerInternal: "true", + objectmeta.ServiceAnnotationAzurePLSCreate: "true", + objectmeta.ServiceAnnotationAzurePLSName: validPLSName, + objectmeta.ServiceAnnotationAzurePLSResourceGroup: altResourceGroup, + }, + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, + getResponses: map[string]*armnetwork.PrivateLinkService{ + altResourceGroup + "/" + validPLSName: {ID: ptr.To(altPLSResourceID)}, + }, + wantPLSID: ptr.To(altPLSResourceID), + }, + { + name: "PLS Get returns object without ID — soft pending, no error, no ID set", + service: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + objectmeta.ServiceAnnotationAzureLoadBalancerInternal: "true", + objectmeta.ServiceAnnotationAzurePLSCreate: "true", + objectmeta.ServiceAnnotationAzurePLSName: validPLSName, + }, + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, + getResponses: map[string]*armnetwork.PrivateLinkService{ + validResourceGroup + "/" + validPLSName: {ID: nil}, + }, + wantPLSID: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &Reconciler{ + AzurePrivateLinkServicesClient: &fakePrivateLinkServicesClient{ + GetResponses: tt.getResponses, + GetErrOverride: tt.getErrOverride, + }, + ResourceGroupName: validResourceGroup, + } + got := &fleetnetv1alpha1.InternalServiceExport{} + err := r.setAzureRelatedPrivateLinkInformation(context.Background(), tt.service, got) + if (err != nil) != tt.wantErr { + t.Fatalf("setAzureRelatedPrivateLinkInformation() error = %v, wantErr %v", err, tt.wantErr) + } + if err != nil { + return + } + if diff := cmp.Diff(tt.wantPLSID, got.Spec.PrivateLinkServiceResourceID); diff != "" { + t.Errorf("PrivateLinkServiceResourceID mismatch (-want, +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/controllers/member/serviceexport/suite_test.go b/pkg/controllers/member/serviceexport/suite_test.go index ae71db48..61f58c9f 100644 --- a/pkg/controllers/member/serviceexport/suite_test.go +++ b/pkg/controllers/member/serviceexport/suite_test.go @@ -130,15 +130,25 @@ var _ = BeforeSuite(func() { }, } + // Fake PLS client seeded with the resource the L7-FrontDoor integration + // spec expects to find. Keyed on "/" matching + // fakePrivateLinkServicesClient.Get. + privateLinkServiceGetResponses := map[string]*armnetwork.PrivateLinkService{ + validResourceGroup + "/" + testPrivateLinkServiceName: { + ID: ptr.To(testPrivateLinkServiceResourceID), + }, + } + err = (&Reconciler{ - MemberClusterID: memberClusterID, - MemberClient: memberClient, - HubClient: hubClient, - HubNamespace: hubNSForMember, - Recorder: ctrlMgr.GetEventRecorderFor(ControllerName), - AzurePublicIPAddressClient: &fakePublicIPAddressClient{ListResponse: publicIPAddressListResponse}, - ResourceGroupName: validResourceGroup, - EnableTrafficManagerFeature: true, + MemberClusterID: memberClusterID, + MemberClient: memberClient, + HubClient: hubClient, + HubNamespace: hubNSForMember, + Recorder: ctrlMgr.GetEventRecorderFor(ControllerName), + AzurePublicIPAddressClient: &fakePublicIPAddressClient{ListResponse: publicIPAddressListResponse}, + AzurePrivateLinkServicesClient: &fakePrivateLinkServicesClient{GetResponses: privateLinkServiceGetResponses}, + ResourceGroupName: validResourceGroup, + EnableTrafficManagerFeature: true, }).SetupWithManager(ctrlMgr) Expect(err).NotTo(HaveOccurred()) diff --git a/test/common/azurefrontdoor/fakeprovider/origin.go b/test/common/azurefrontdoor/fakeprovider/origin.go new file mode 100644 index 00000000..ed2bf5bf --- /dev/null +++ b/test/common/azurefrontdoor/fakeprovider/origin.go @@ -0,0 +1,137 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package fakeprovider + +import ( + "context" + "fmt" + "net/http" + "sync" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + azcorefake "github.com/Azure/azure-sdk-for-go/sdk/azcore/fake" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cdn/armcdn/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cdn/armcdn/v2/fake" + "k8s.io/utils/ptr" +) + +// OriginResourceIDFormat is the ARM ID format for a single Azure Front Door +// origin under a (profile, originGroup). The FrontDoorBackend reconciler +// copies this into FrontDoorOriginStatus.ResourceID. +const OriginResourceIDFormat = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Cdn/profiles/%s/originGroups/%s/origins/%s" + +type originKey struct { + profile, group, origin string +} + +type originStore struct { + mu sync.Mutex + origins map[originKey]armcdn.AFDOrigin +} + +func newOriginStore() *originStore { + return &originStore{origins: map[originKey]armcdn.AFDOrigin{}} +} + +// OriginFake bundles the client with a handle to the underlying store so +// tests can peek/mutate state directly. +type OriginFake struct { + Client *armcdn.AFDOriginsClient + store *originStore +} + +// NewOriginFake returns an armcdn.AFDOriginsClient backed by an in-memory +// fake with its own independent state. +func NewOriginFake() (*OriginFake, error) { + store := newOriginStore() + srv := fake.AFDOriginsServer{ + Get: store.get, + BeginCreate: store.beginCreate, + BeginDelete: store.beginDelete, + } + factory, err := armcdn.NewClientFactory(DefaultSubscriptionID, &azcorefake.TokenCredential{}, + &arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: fake.NewAFDOriginsServerTransport(&srv), + }, + }) + if err != nil { + return nil, err + } + return &OriginFake{Client: factory.NewAFDOriginsClient(), store: store}, nil +} + +// Count returns the number of origins currently programmed under a given +// (profile, originGroup). Tests use this to assert per-cluster origin +// creation without pinning specific names. +func (f *OriginFake) Count(profileName, groupName string) int { + f.store.mu.Lock() + defer f.store.mu.Unlock() + var n int + for k := range f.store.origins { + if k.profile == profileName && k.group == groupName { + n++ + } + } + return n +} + +// Get returns a snapshot of the origin with the given key, or false when it +// is not present. Tests may assert on Weight / SharedPrivateLinkResource / +// HostName. +func (f *OriginFake) Get(profileName, groupName, originName string) (armcdn.AFDOrigin, bool) { + f.store.mu.Lock() + defer f.store.mu.Unlock() + o, ok := f.store.origins[originKey{profileName, groupName, originName}] + return o, ok +} + +func (s *originStore) get(_ context.Context, resourceGroupName string, profileName string, groupName string, originName string, _ *armcdn.AFDOriginsClientGetOptions) (resp azcorefake.Responder[armcdn.AFDOriginsClientGetResponse], errResp azcorefake.ErrorResponder) { + if resourceGroupName != DefaultResourceGroupName { + errResp.SetResponseError(http.StatusForbidden, "AuthorizationFailed") + return resp, errResp + } + s.mu.Lock() + defer s.mu.Unlock() + o, ok := s.origins[originKey{profileName, groupName, originName}] + if !ok { + errResp.SetResponseError(http.StatusNotFound, "NotFound") + return resp, errResp + } + resp.SetResponse(http.StatusOK, armcdn.AFDOriginsClientGetResponse{AFDOrigin: o}, nil) + return resp, errResp +} + +func (s *originStore) beginCreate(_ context.Context, resourceGroupName string, profileName string, groupName string, originName string, parameters armcdn.AFDOrigin, _ *armcdn.AFDOriginsClientBeginCreateOptions) (resp azcorefake.PollerResponder[armcdn.AFDOriginsClientCreateResponse], errResp azcorefake.ErrorResponder) { + if resourceGroupName != DefaultResourceGroupName { + errResp.SetResponseError(http.StatusForbidden, "AuthorizationFailed") + return resp, errResp + } + created := parameters + created.Name = ptr.To(originName) + created.ID = ptr.To(fmt.Sprintf(OriginResourceIDFormat, DefaultSubscriptionID, DefaultResourceGroupName, profileName, groupName, originName)) + + s.mu.Lock() + s.origins[originKey{profileName, groupName, originName}] = created + s.mu.Unlock() + + resp.SetTerminalResponse(http.StatusOK, armcdn.AFDOriginsClientCreateResponse{AFDOrigin: created}, nil) + return resp, errResp +} + +func (s *originStore) beginDelete(_ context.Context, resourceGroupName string, profileName string, groupName string, originName string, _ *armcdn.AFDOriginsClientBeginDeleteOptions) (resp azcorefake.PollerResponder[armcdn.AFDOriginsClientDeleteResponse], errResp azcorefake.ErrorResponder) { + if resourceGroupName != DefaultResourceGroupName { + errResp.SetResponseError(http.StatusForbidden, "AuthorizationFailed") + return resp, errResp + } + s.mu.Lock() + delete(s.origins, originKey{profileName, groupName, originName}) + s.mu.Unlock() + + resp.SetTerminalResponse(http.StatusOK, armcdn.AFDOriginsClientDeleteResponse{}, nil) + return resp, errResp +} diff --git a/test/common/azurefrontdoor/fakeprovider/origingroup.go b/test/common/azurefrontdoor/fakeprovider/origingroup.go new file mode 100644 index 00000000..73fff523 --- /dev/null +++ b/test/common/azurefrontdoor/fakeprovider/origingroup.go @@ -0,0 +1,125 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package fakeprovider + +import ( + "context" + "fmt" + "net/http" + "sync" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + azcorefake "github.com/Azure/azure-sdk-for-go/sdk/azcore/fake" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cdn/armcdn/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cdn/armcdn/v2/fake" + "k8s.io/utils/ptr" +) + +// OriginGroupResourceIDFormat mirrors the ARM ID an AFD origin group carries +// in the real Azure response. The FrontDoorBackend reconciler copies this ID +// into FrontDoorBackendStatus.OriginGroupResourceID; leaving it empty would +// mask bugs in the status-population path. +const OriginGroupResourceIDFormat = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Cdn/profiles/%s/originGroups/%s" + +// originGroupKey scopes state by (profile, group) so a single fake client +// can host origin groups belonging to sibling profiles simultaneously. +type originGroupKey struct { + profile, group string +} + +type originGroupStore struct { + mu sync.Mutex + groups map[originGroupKey]armcdn.AFDOriginGroup +} + +func newOriginGroupStore() *originGroupStore { + return &originGroupStore{groups: map[originGroupKey]armcdn.AFDOriginGroup{}} +} + +// OriginGroupFake bundles the client with a handle to the underlying store so +// tests can peek/mutate state directly (mirrors WAFPolicyFake shape). +type OriginGroupFake struct { + Client *armcdn.AFDOriginGroupsClient + store *originGroupStore +} + +// NewOriginGroupFake returns an armcdn.AFDOriginGroupsClient backed by an +// in-memory fake with its own independent state. +func NewOriginGroupFake() (*OriginGroupFake, error) { + store := newOriginGroupStore() + srv := fake.AFDOriginGroupsServer{ + Get: store.get, + BeginCreate: store.beginCreate, + BeginDelete: store.beginDelete, + } + factory, err := armcdn.NewClientFactory(DefaultSubscriptionID, &azcorefake.TokenCredential{}, + &arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: fake.NewAFDOriginGroupsServerTransport(&srv), + }, + }) + if err != nil { + return nil, err + } + return &OriginGroupFake{Client: factory.NewAFDOriginGroupsClient(), store: store}, nil +} + +// Has reports whether an OriginGroup with the given (profile, name) exists. +// Tests use this to assert deletion + creation without touching Azure ARM +// semantics directly. +func (f *OriginGroupFake) Has(profileName, groupName string) bool { + f.store.mu.Lock() + defer f.store.mu.Unlock() + _, ok := f.store.groups[originGroupKey{profileName, groupName}] + return ok +} + +func (s *originGroupStore) get(_ context.Context, resourceGroupName string, profileName string, groupName string, _ *armcdn.AFDOriginGroupsClientGetOptions) (resp azcorefake.Responder[armcdn.AFDOriginGroupsClientGetResponse], errResp azcorefake.ErrorResponder) { + if resourceGroupName != DefaultResourceGroupName { + errResp.SetResponseError(http.StatusForbidden, "AuthorizationFailed") + return resp, errResp + } + s.mu.Lock() + defer s.mu.Unlock() + og, ok := s.groups[originGroupKey{profileName, groupName}] + if !ok { + errResp.SetResponseError(http.StatusNotFound, "NotFound") + return resp, errResp + } + resp.SetResponse(http.StatusOK, armcdn.AFDOriginGroupsClientGetResponse{AFDOriginGroup: og}, nil) + return resp, errResp +} + +func (s *originGroupStore) beginCreate(_ context.Context, resourceGroupName string, profileName string, groupName string, parameters armcdn.AFDOriginGroup, _ *armcdn.AFDOriginGroupsClientBeginCreateOptions) (resp azcorefake.PollerResponder[armcdn.AFDOriginGroupsClientCreateResponse], errResp azcorefake.ErrorResponder) { + if resourceGroupName != DefaultResourceGroupName { + errResp.SetResponseError(http.StatusForbidden, "AuthorizationFailed") + return resp, errResp + } + created := parameters + created.Name = ptr.To(groupName) + created.ID = ptr.To(fmt.Sprintf(OriginGroupResourceIDFormat, DefaultSubscriptionID, DefaultResourceGroupName, profileName, groupName)) + + s.mu.Lock() + s.groups[originGroupKey{profileName, groupName}] = created + s.mu.Unlock() + + resp.SetTerminalResponse(http.StatusOK, armcdn.AFDOriginGroupsClientCreateResponse{AFDOriginGroup: created}, nil) + return resp, errResp +} + +func (s *originGroupStore) beginDelete(_ context.Context, resourceGroupName string, profileName string, groupName string, _ *armcdn.AFDOriginGroupsClientBeginDeleteOptions) (resp azcorefake.PollerResponder[armcdn.AFDOriginGroupsClientDeleteResponse], errResp azcorefake.ErrorResponder) { + if resourceGroupName != DefaultResourceGroupName { + errResp.SetResponseError(http.StatusForbidden, "AuthorizationFailed") + return resp, errResp + } + s.mu.Lock() + delete(s.groups, originGroupKey{profileName, groupName}) + s.mu.Unlock() + + resp.SetTerminalResponse(http.StatusOK, armcdn.AFDOriginGroupsClientDeleteResponse{}, nil) + return resp, errResp +}