diff --git a/apiserver/deps.txt b/apiserver/deps.txt index 94d10a0d9ec..14f5e407175 100644 --- a/apiserver/deps.txt +++ b/apiserver/deps.txt @@ -78,6 +78,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/net v0.57.0 diff --git a/app-policy/deps.txt b/app-policy/deps.txt index 1009e8fb014..eb88a080db4 100644 --- a/app-policy/deps.txt +++ b/app-policy/deps.txt @@ -81,6 +81,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/net v0.57.0 diff --git a/calicoctl/calicoctl/commands/ipam/show.go b/calicoctl/calicoctl/commands/ipam/show.go index 963bd2fb86b..435316fdfbe 100644 --- a/calicoctl/calicoctl/commands/ipam/show.go +++ b/calicoctl/calicoctl/commands/ipam/show.go @@ -17,7 +17,6 @@ package ipam import ( "context" "fmt" - "math" "os" "reflect" "sort" @@ -209,37 +208,41 @@ func ShowBlockUtilization(ctx context.Context, ipamClient ipam.Interface, showBl } t := table.NewWriter() t.SetOutputMirror(os.Stdout) - t.AppendHeader(table.Row{"GROUPING", "CIDR", "IPS TOTAL", "IPS IN USE", "IPS FREE"}) + t.AppendHeader(table.Row{"GROUPING", "CIDR", "IPS TOTAL", "IPS IN USE", "IPS RESERVED", "IPS FREE"}) t.SetColumnConfigs([]table.ColumnConfig{ {Name: "IPS TOTAL", Align: text.AlignRight}, {Name: "IPS IN USE", Align: text.AlignRight}, + {Name: "IPS RESERVED", Align: text.AlignRight}, {Name: "IPS FREE", Align: text.AlignRight}, }) - genRow := func(kind, cidr string, inUse, capacity float64) table.Row { + // IN USE counts allocated IPs and RESERVED counts IPs that an IPReservation + // covers; an IP allocated before it was reserved falls into both, so the + // percentages need not add up to 100. FREE counts the IPs that are neither, + // which is why it comes from the library rather than being derived here. + genRow := func(kind, cidr string, capacity, inUse, reserved, free int) table.Row { + withPercentage := func(n int) string { + return fmt.Sprintf("%.5g (%.f%%)", float64(n), 100*float64(n)/float64(capacity)) + } return table.Row{ kind, cidr, - fmt.Sprintf("%.5g", capacity), - // Note: the '+capacity/2' bits here give us rounding to the nearest - // integer, instead of rounding down, and so ensure that the two percentages - // add up to 100. - fmt.Sprintf("%.5g (%.f%%)", inUse, 100*inUse/capacity), - fmt.Sprintf("%.5g (%.f%%)", capacity-inUse, 100*(capacity-inUse)/capacity), + fmt.Sprintf("%.5g", float64(capacity)), + withPercentage(inUse), + withPercentage(reserved), + withPercentage(free), } } for _, poolUse := range usage { var blockRows []table.Row - var poolInUse float64 for _, blockUse := range poolUse.Blocks { - blockRows = append(blockRows, genRow("Block", blockUse.CIDR.String(), float64(blockUse.Capacity-blockUse.Available), float64(blockUse.Capacity))) - poolInUse += float64(blockUse.Capacity - blockUse.Available) + blockRows = append(blockRows, genRow("Block", blockUse.CIDR.String(), + blockUse.Capacity, blockUse.InUse, blockUse.Reserved, blockUse.Available)) } - ones, bits := poolUse.CIDR.Mask.Size() - poolCapacity := math.Pow(2, float64(bits-ones)) - if ones > 0 { + if ones, _ := poolUse.CIDR.Mask.Size(); ones > 0 { // Only show the IP Pool row for a real IP Pool and not for the orphaned // block case. - t.AppendRow(genRow("IP Pool", poolUse.CIDR.String(), poolInUse, poolCapacity)) + t.AppendRow(genRow("IP Pool", poolUse.CIDR.String(), + poolUse.Capacity, poolUse.InUse, poolUse.Reserved, poolUse.Available)) } if showBlocks { t.AppendRows(blockRows) diff --git a/calicoctl/deps.txt b/calicoctl/deps.txt index ce36cacae76..b4694840962 100644 --- a/calicoctl/deps.txt +++ b/calicoctl/deps.txt @@ -75,6 +75,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/net v0.57.0 diff --git a/charts/calico/templates/calico-kube-controllers-rbac.yaml b/charts/calico/templates/calico-kube-controllers-rbac.yaml index 06aba81caef..29b938f9488 100644 --- a/charts/calico/templates/calico-kube-controllers-rbac.yaml +++ b/charts/calico/templates/calico-kube-controllers-rbac.yaml @@ -121,6 +121,7 @@ rules: - ipreservations verbs: - list + - watch - apiGroups: ["projectcalico.org", "crd.projectcalico.org"] resources: - blockaffinities diff --git a/cmd/deps.txt b/cmd/deps.txt index 360101bb377..3eda8055af0 100644 --- a/cmd/deps.txt +++ b/cmd/deps.txt @@ -174,6 +174,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/net v0.57.0 diff --git a/cni-plugin/deps.txt b/cni-plugin/deps.txt index 4595cee980f..9cf6520989f 100644 --- a/cni-plugin/deps.txt +++ b/cni-plugin/deps.txt @@ -80,6 +80,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/mod v0.38.0 diff --git a/confd/deps.txt b/confd/deps.txt index c7fd35ef50e..a2d9508efab 100644 --- a/confd/deps.txt +++ b/confd/deps.txt @@ -63,6 +63,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/net v0.57.0 diff --git a/design/ipam/ipam-core-library.md b/design/ipam/ipam-core-library.md index b0a8ed32a2b..09b35747f87 100644 --- a/design/ipam/ipam-core-library.md +++ b/design/ipam/ipam-core-library.md @@ -20,6 +20,14 @@ here. A few methods carry design-relevant constraints worth calling out: - **`AutoAssign`** returns block-masked CIDRs, not `/32` (or `/128`). Callers narrow at the boundary. This is load-bearing for the CNI plugin's per-block route programming. - **`AssignIP`** enforces the target pool's `allowedUses` when `AssignIPArgs.IntendedUse` is non-empty: it fails if the pool containing the requested IP does not permit that use, mirroring the `filterPoolsByUse` filter `AutoAssign` applies. Callers that leave `IntendedUse` empty are exempt (back-compat). This closes a gap where a specific-IP request (e.g. the CNI `ipAddrs` annotation) could draw from a pool not sanctioned for its use. +- **`GetUtilization`** reports `Capacity`, `InUse`, `Reserved` and `Available` per pool and per block. `InUse` (allocated) and `Reserved` (covered by an `IPReservation`) overlap when an + address was allocated before it was reserved, so `Capacity` is not their sum plus `Available`; `Available` counts addresses that are neither, and is the only one of the four that + answers "how many can still be handed out". Consumers must read it rather than deriving it. Pool-level counts span the whole pool CIDR, including space no block covers yet - a + reservation over unblocked space is still unassignable - so they are computed as a set operation (pool minus reservations minus blocks, via `go4.org/netipx`) rather than summed + from the blocks. Reservations may overlap and nest arbitrarily, which is why a set is needed and not a sum over CIDRs. +- **`NumReservedIPsInCIDR`** is the pool-level reserved count on its own, for callers that already hold the `IPReservation`s and would rather not pay for a list of every allocation + block. kube-controllers uses it for `ipam_ippool_reserved` from syncer-fed reservations (see [ipam-gc](./ipam-gc.md#metrics)). It takes the resources, not CIDRs, so that a variant + can take a second kind of reserving resource without reshaping its callers. - **`ReleaseIPs`** takes `ReleaseOptions` with a sequence number; every release path must plumb it through (see [CAS retry and sequence numbers](#cas-retry-and-sequence-numbers)). - **`SetOwnerAttributes`** is KubeVirt-only and swaps owner attributes under preconditions, without releasing and re-allocating. Felix's live-migration monitor is the only non-CNI caller. @@ -32,6 +40,10 @@ here. A few methods carry design-relevant constraints worth calling out: - Don't leak `crd.projectcalico.org/v1` types through new public APIs. The `lib/v3` -> `lib/internalapi` rename (https://github.com/projectcalico/calico/pull/11870) exists to keep that boundary clean. - `AutoAssign` returning block-masked CIDRs is load-bearing for the CNI plugin's routing. Don't quietly switch to `/32`. +- Anything that makes an address unassignable has to be discounted by `GetUtilization` as well as by the allocation path, or the reporting surfaces over-count free addresses. The + two must be fed from the same set of reserved CIDRs: allocation and the per-block counts share the `addrFilter`, and the pool-level counts use the same CIDRs as a set. +- There is one implementation of the reserved-set arithmetic, in [`reserved.go`](../../libcalico-go/lib/ipam/reserved.go). `GetUtilization` and `NumReservedIPsInCIDR` are both thin + callers of it. Don't grow a second copy in a consumer - a reporting surface that disagrees with `calicoctl ipam show` is worse than no surface. ## AutoAssign and host affinity diff --git a/design/ipam/ipam-datastore.md b/design/ipam/ipam-datastore.md index fbd32d38169..e8d48fe0a8f 100644 --- a/design/ipam/ipam-datastore.md +++ b/design/ipam/ipam-datastore.md @@ -83,7 +83,8 @@ the block syncer plus a handle-side scan rather than a watch. See https://github Both stored as CRDs alongside the other IPAM resources. [`ipam_config.go`](../../libcalico-go/lib/backend/k8s/resources/ipam_config.go) wraps the singleton `IPAMConfig`. Storage shape only - field semantics, defaults, and `StrictAffinity` / `MaxBlocksPerHost` / `AutoAllocateBlocks` interactions live in -[`ipam-core-library.md`](./ipam-core-library.md#ipamconfig). `IPReservation` is read at allocation time and converted into an ordinal filter; never participates in CAS. +[`ipam-core-library.md`](./ipam-core-library.md#ipamconfig). `IPReservation` is read at allocation time and converted into an ordinal filter, and again by `GetUtilization` so that +the reporting surfaces don't count reserved addresses as free. kube-controllers watches it on its syncer instead, to keep the read off the IPAM sync loop. Never participates in CAS. **Review notes** diff --git a/design/ipam/ipam-gc.md b/design/ipam/ipam-gc.md index 2f8c30809cf..6c596ee848c 100644 --- a/design/ipam/ipam-gc.md +++ b/design/ipam/ipam-gc.md @@ -214,11 +214,23 @@ variants exist for backward compatibility. `updateMetrics` recomputes from scratch every sync - one walk over all blocks, no incremental state. The full-recompute *is* the consistency check; switching to incremental updates without a separate consistency check loses the protection. +`ipam_ippool_reserved` is the exception to that walk: reservations make addresses unassignable without allocating them, and can cover pool space no block has been carved from, so the +number isn't in the block state the controller tracks. `IPReservation` is therefore a fourth kind on the controller's syncer, cached by name in `reservations`, and +`updateReservedMetrics` counts the covered addresses per pool with `ipam.NumReservedIPsInCIDR` (see [ipam-core-library](./ipam-core-library.md#public-api-surface)). The arithmetic is +the library's, so the gauge agrees with `calicoctl ipam show`; the input is the syncer's, so the sync loop makes no datastore request for it. Being per-pool rather than per-node, the +gauge is labelled `ippool` only, like `ipam_ippool_size`. It may overlap `ipam_allocations_in_use`, so usable capacity is +`ipam_ippool_size - ipam_allocations_in_use - ipam_ippool_reserved` only when no reserved address is also allocated. + +Watching `IPReservation` needs `watch` in the kube-controllers ClusterRole, in the chart **and** in tigera/operator. With only `list` granted the List still succeeds and the syncer +still reaches in-sync, so the symptom is a hot re-list of `IPReservation`s rather than a stalled controller - easy to miss in review, noisy in production. + **Review notes** - `ipam_allocations_gc_candidates > 0` for extended periods is the canonical "GC is stuck" signal. Alert on it. - `ipam_allocations_gc_reclamations` rate is the canonical "we have a real leak somewhere" signal. Alert on it. - Don't switch `updateMetrics` to incremental updates without a separate consistency check. The current full-recompute is the consistency check. +- A metric is not a licence to add a datastore request to the sync loop. The loop shares a goroutine with leak GC, and past overload has clogged it; new inputs belong on the syncer. + `ipam_ippool_reserved` was caught doing a LIST of every block per sync in review (https://github.com/projectcalico/calico/pull/13331). - The in-memory state maps must agree at all times. `assertConsistentState` in `ipam_test.go` is the canonical invariant check; any new map mutation needs a test that exercises it. The v3.32 memory-leak family (https://github.com/projectcalico/calico/pull/12277, /12286, /12287, /12288) all came from "added to one path, forgot another." diff --git a/design/ipam/ipam-other-callers.md b/design/ipam/ipam-other-callers.md index f53451a21a5..12004241b65 100644 --- a/design/ipam/ipam-other-callers.md +++ b/design/ipam/ipam-other-callers.md @@ -23,10 +23,15 @@ migrate` parses the tunnel prefixes, and the CNI DEL path releases by both the h IPAM-meaningful subcommands; `configure` and `split` are admin-CRUD on `IPAMConfig` / `IPPool`. The `check` algorithm reuses the validity heuristics the GC applies, but exposed for manual review without a running controller. +`show` prints IPS TOTAL / IN USE / RESERVED / FREE per pool and per block, taking all four straight from `GetUtilization` rather than deriving any of them: IN USE and RESERVED can +cover the same address, so the columns need not sum to the total, and FREE is the only column that means "still assignable". See +[ipam-core-library](./ipam-core-library.md#public-api-surface). + **Review notes** - New tunnel handle prefixes need to be added to `calicoctl datastore migrate`, which rewrites tunnel handle IDs during node renames. - `check` and the GC share validity logic. If you change one, check the other doesn't drift. +- The `show` columns are a reporting surface, not a derivation. If a new mechanism withholds addresses, it has to reach `GetUtilization` or `show` silently over-counts FREE. ## Node tunnel-address allocator diff --git a/e2e/deps.txt b/e2e/deps.txt index 7a4529f58a4..fdeb064e8ba 100644 --- a/e2e/deps.txt +++ b/e2e/deps.txt @@ -95,6 +95,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/mod v0.38.0 diff --git a/felix/deps.txt b/felix/deps.txt index dd40688228f..5fd632154ee 100644 --- a/felix/deps.txt +++ b/felix/deps.txt @@ -128,6 +128,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/mod v0.38.0 diff --git a/go.mod b/go.mod index b5a262b89bf..0036bd2408c 100644 --- a/go.mod +++ b/go.mod @@ -92,6 +92,7 @@ require ( go.etcd.io/etcd/client/v2 v2.305.31 go.etcd.io/etcd/client/v3 v3.6.12 go.yaml.in/yaml/v3 v3.0.4 + go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/mod v0.38.0 golang.org/x/net v0.57.0 golang.org/x/oauth2 v0.36.0 diff --git a/go.sum b/go.sum index 35b505e89ad..30437f3b2ba 100644 --- a/go.sum +++ b/go.sum @@ -890,6 +890,8 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/hack/deps.txt b/hack/deps.txt index 4e2b8a31646..0b2fdc4958c 100644 --- a/hack/deps.txt +++ b/hack/deps.txt @@ -62,6 +62,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/mod v0.38.0 diff --git a/kube-controllers/deps.txt b/kube-controllers/deps.txt index e1aaa706285..1675334cc9e 100644 --- a/kube-controllers/deps.txt +++ b/kube-controllers/deps.txt @@ -106,6 +106,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/mod v0.38.0 diff --git a/kube-controllers/pkg/controllers/node/ipam.go b/kube-controllers/pkg/controllers/node/ipam.go index 4eee71920c3..fba789a1ddd 100644 --- a/kube-controllers/pkg/controllers/node/ipam.go +++ b/kube-controllers/pkg/controllers/node/ipam.go @@ -17,8 +17,10 @@ package node import ( "context" "fmt" + "maps" "math" "net" + "slices" "strings" "time" @@ -62,6 +64,7 @@ var ( // Single dimension metrics. Legacy metrics are replaced by multidimensional equivalents above. Retain for // backwards compatibility. poolSizeGauge *prometheus.GaugeVec + poolReservedGauge *prometheus.GaugeVec legacyAllocationsGauge *prometheus.GaugeVec legacyBlocksGauge *prometheus.GaugeVec legacyBorrowedGauge *prometheus.GaugeVec @@ -98,6 +101,14 @@ func init() { }, []string{"ippool"}) prometheus.MustRegister(poolSizeGauge) + poolReservedGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ipam_ippool_reserved", + // Careful: the metrics FV asserts on substrings of this output, so naming + // another metric here would make its absence checks match this help text. + Help: "Number of addresses in the IP Pool that an IPReservation covers, and so cannot be assigned.", + }, []string{"ippool"}) + prometheus.MustRegister(poolReservedGauge) + // Total IP allocations. legacyAllocationsGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "ipam_allocations_per_node", @@ -171,6 +182,7 @@ func NewIPAMController(cfg config.NodeControllerConfig, c client.Interface, cs k syncerUpdates: make(chan any, utils.BatchUpdateSize), allBlocks: make(map[string]model.KVPair), + reservations: make(map[string]*apiv3.IPReservation), allocationsByBlock: make(map[string]map[string]*allocation), allocationState: newAllocationState(), handleTracker: newHandleTracker(), @@ -222,6 +234,11 @@ type IPAMController struct { // Raw block storage, keyed by CIDR. allBlocks map[string]model.KVPair + // IPReservations, keyed by name. They make addresses unassignable without + // allocating them, so the block state above cannot account for them; only the + // reserved-IP metric needs them. + reservations map[string]*apiv3.IPReservation + // allocationState is the primary in-memory representation of IPAM allocations used by the garbage collector. allocationState *allocationState @@ -326,7 +343,7 @@ func (c *IPAMController) onUpdate(update bapi.Update) { switch update.Key.(type) { case model.ResourceKey: switch update.KVPair.Key.(model.ResourceKey).Kind { - case internalapi.KindNode, apiv3.KindIPPool, apiv3.KindClusterInformation: + case internalapi.KindNode, apiv3.KindIPPool, apiv3.KindIPReservation, apiv3.KindClusterInformation: c.syncerUpdates <- update.KVPair } case model.BlockKey: @@ -446,6 +463,9 @@ func (c *IPAMController) handleUpdate(upd any) { case apiv3.KindIPPool: c.handlePoolUpdate(upd) return + case apiv3.KindIPReservation: + c.handleIPReservationUpdate(upd) + return case apiv3.KindClusterInformation: c.handleClusterInformationUpdate(upd) return @@ -507,6 +527,18 @@ func (c *IPAMController) handlePoolUpdate(kvp model.KVPair) { } } +// handleIPReservationUpdate wraps up the logic to execute when receiving an +// IPReservation update. We track reservations only to report how much of each pool +// they cover; see updateReservedMetrics. +func (c *IPAMController) handleIPReservationUpdate(kvp model.KVPair) { + name := kvp.Key.(model.ResourceKey).Name + if kvp.Value != nil { + c.reservations[name] = kvp.Value.(*apiv3.IPReservation) + } else { + delete(c.reservations, name) + } +} + // handleClusterInformationUpdate wraps the logic to execute when receiving a clusterinformation update. func (c *IPAMController) handleClusterInformationUpdate(kvp model.KVPair) { if kvp.Value != nil { @@ -679,7 +711,7 @@ func (c *IPAMController) onPoolUpdated(pool *apiv3.IPPool) { func (c *IPAMController) onPoolDeleted(poolName string) { unregisterMetricVectorsForPool(poolName) - clearPoolSizeMetric(poolName) + clearPoolMetrics(poolName) c.poolManager.onPoolDeleted(poolName) } @@ -766,9 +798,35 @@ func (c *IPAMController) updateMetrics() { for node, num := range legacyBorrowedIPsByNode { legacyBorrowedGauge.WithLabelValues(node).Set(float64(num)) } + + c.updateReservedMetrics() + log.Debug("IPAM metrics updated") } +// updateReservedMetrics publishes how much of each pool an IPReservation covers. +// Unlike the counts above, this cannot be derived from the blocks we track: a +// reservation makes addresses unassignable without allocating them, and it can cover +// pool space that no block has been carved from yet. The reservations come from the +// syncer like everything else here, so this needs no datastore reads; the arithmetic +// is the library's, so this agrees with what `calicoctl ipam show` reports. +func (c *IPAMController) updateReservedMetrics() { + reservations := slices.Collect(maps.Values(c.reservations)) + for poolName, pool := range c.poolManager.allPools { + _, poolCIDR, err := cnet.ParseCIDR(pool.Spec.CIDR) + if err != nil { + log.WithError(err).Warnf("Unable to parse CIDR for IP Pool %s; skipping its reserved-IP metric", poolName) + continue + } + numReserved, err := ipam.NumReservedIPsInCIDR(*poolCIDR, reservations) + if err != nil { + log.WithError(err).Warnf("Unable to count reserved IPs in IP Pool %s", poolName) + continue + } + poolReservedGauge.With(prometheus.Labels{"ippool": poolName}).Set(float64(numReserved)) + } +} + // releaseUnusedBlocks looks at known empty blocks, and releases their affinity // if appropriate. A block is a candidate for having its affinity released if: // @@ -1615,8 +1673,9 @@ func publishPoolSizeMetric(pool *apiv3.IPPool) { poolSizeGauge.With(prometheus.Labels{"ippool": pool.Name}).Set(poolSize) } -func clearPoolSizeMetric(poolName string) { +func clearPoolMetrics(poolName string) { poolSizeGauge.Delete(prometheus.Labels{"ippool": poolName}) + poolReservedGauge.Delete(prometheus.Labels{"ippool": poolName}) } // When we stop tracking a node, clear counters to prevent accumulation of stale metrics. diff --git a/kube-controllers/pkg/controllers/node/ipam_test.go b/kube-controllers/pkg/controllers/node/ipam_test.go index bdfb2225336..1a39433fbfb 100644 --- a/kube-controllers/pkg/controllers/node/ipam_test.go +++ b/kube-controllers/pkg/controllers/node/ipam_test.go @@ -23,6 +23,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -234,6 +236,49 @@ var _ = Describe("IPAM controller UTs", func() { done() }) + It("should publish the reserved-IP gauge from syncer updates", func() { + c.Start(stopChan) + resume := c.pause() + defer resume() + + poolReservedGauge.Reset() + poolName := "reserved-gauge-test-pool" + reservedGauge := func() float64 { + c.updateReservedMetrics() + return testutil.ToFloat64(poolReservedGauge.With(prometheus.Labels{"ippool": poolName})) + } + + c.handleUpdate(model.KVPair{ + Key: model.ResourceKey{Kind: apiv3.KindIPPool, Name: poolName}, + Value: &apiv3.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: poolName}, + Spec: apiv3.IPPoolSpec{CIDR: "10.0.0.0/24"}, + }, + }) + Expect(reservedGauge()).To(BeZero(), "no reservations yet") + + // No block covers this space, so the count cannot come from the block state + // the controller tracks. The two reservations overlap, so the shared /29 + // must only be counted once. + reservationKey := model.ResourceKey{Kind: apiv3.KindIPReservation, Name: "test-reservation"} + c.handleUpdate(model.KVPair{ + Key: reservationKey, + Value: &apiv3.IPReservation{ + ObjectMeta: metav1.ObjectMeta{Name: reservationKey.Name}, + Spec: apiv3.IPReservationSpec{ReservedCIDRs: []string{"10.0.0.128/28", "10.0.0.128/29"}}, + }, + }) + Expect(reservedGauge()).To(Equal(16.0)) + + // Deleting the reservation frees the addresses again. + c.handleUpdate(model.KVPair{Key: reservationKey}) + Expect(reservedGauge()).To(BeZero()) + + // Deleting the pool should take its gauge with it. + c.onPoolDeleted(poolName) + Expect(testutil.CollectAndCount(poolReservedGauge)).To(BeZero()) + }) + Describe("VMI allocation validation", func() { makeVMIAllocation := func(ns, vmName string) *allocation { return &allocation{ diff --git a/kube-controllers/pkg/controllers/node/metrics_fv_test.go b/kube-controllers/pkg/controllers/node/metrics_fv_test.go index 1654c753dde..b0aa883f43e 100644 --- a/kube-controllers/pkg/controllers/node/metrics_fv_test.go +++ b/kube-controllers/pkg/controllers/node/metrics_fv_test.go @@ -187,6 +187,10 @@ var _ = Describe("kube-controllers metrics FV tests", Ordered, ContinueOnFailure `ipam_ippool_size{ippool="test-ippool-1"} 256`, `ipam_ippool_size{ippool="test-ippool-2"} 65536`, `ipam_ippool_size{ippool="test-ippool-3"} 256`, + // No IPReservations yet, so nothing is reserved. + `ipam_ippool_reserved{ippool="test-ippool-1"} 0`, + `ipam_ippool_reserved{ippool="test-ippool-2"} 0`, + `ipam_ippool_reserved{ippool="test-ippool-3"} 0`, `ipam_allocations_in_use{ippool="test-ippool-1",node="node-a"} 0`, `ipam_allocations_in_use{ippool="test-ippool-1",node="node-b"} 0`, `ipam_allocations_in_use{ippool="test-ippool-1",node="node-c"} 0`, @@ -637,6 +641,46 @@ var _ = Describe("kube-controllers metrics FV tests", Ordered, ContinueOnFailure return nil }, time.Second*10, 500*time.Millisecond).Should(BeNil()) }) + + It("should export the reserved-IP metric for an IPReservation", func() { + poolName := "test-ippool-reserved" + createIPPool(poolName, "10.17.0.0/24", calicoClient) + + validateExpectedAndUnexpectedMetrics( + []string{ + fmt.Sprintf(`ipam_ippool_size{ippool=%q} 256`, poolName), + fmt.Sprintf(`ipam_ippool_reserved{ippool=%q} 0`, poolName), + }, + nil, + kubeControllers.IP, + 30*time.Second, 1*time.Second, + ) + + // The reservation is the only thing that changes from here, so the metric + // updating proves the controller watches IPReservations. It covers pool + // space that no block has been carved from, which is the case the metric + // cannot get from the controller's block state. + reservationName := "test-reservation" + createIPReservation(reservationName, []string{"10.17.0.0/28"}, calicoClient) + + validateExpectedAndUnexpectedMetrics( + []string{fmt.Sprintf(`ipam_ippool_reserved{ippool=%q} 16`, poolName)}, + nil, + kubeControllers.IP, + 30*time.Second, 1*time.Second, + ) + + // And deleting it frees the addresses again. + _, err := calicoClient.IPReservations().Delete(context.Background(), reservationName, options.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + validateExpectedAndUnexpectedMetrics( + []string{fmt.Sprintf(`ipam_ippool_reserved{ippool=%q} 0`, poolName)}, + nil, + kubeControllers.IP, + 30*time.Second, 1*time.Second, + ) + }) }) // getMetrics hits the provided prometheus metrics URL and returns the response body @@ -676,6 +720,14 @@ func createIPPool(name string, cidr string, calicoClient client.Interface) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) } +func createIPReservation(name string, cidrs []string, calicoClient client.Interface) { + r := api.NewIPReservation() + r.Name = name + r.Spec.ReservedCIDRs = cidrs + _, err := calicoClient.IPReservations().Create(context.Background(), r, options.SetOptions{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) +} + func allocatePodIPWithHandle(ip string, handle string, node string, pod string, existingOffset int, calicoClient client.Interface) { attrs := map[string]string{"node": node, "pod": pod, "namespace": "default"} err := calicoClient.IPAM().AssignIP(context.Background(), ipam.AssignIPArgs{ diff --git a/kube-controllers/pkg/controllers/utils/syncer.go b/kube-controllers/pkg/controllers/utils/syncer.go index ff8f96a2a82..3d6bd4f9d00 100644 --- a/kube-controllers/pkg/controllers/utils/syncer.go +++ b/kube-controllers/pkg/controllers/utils/syncer.go @@ -51,6 +51,9 @@ func NewDataFeed(c client.Interface, dataStore string) *DataFeed { { ListInterface: model.ResourceListOptions{Kind: apiv3.KindIPPool}, }, + { + ListInterface: model.ResourceListOptions{Kind: apiv3.KindIPReservation}, + }, { ListInterface: model.ResourceListOptions{Kind: apiv3.KindHostEndpoint}, }, diff --git a/libcalico-go/deps.txt b/libcalico-go/deps.txt index 8d829aea36a..cdf0eb88d9c 100644 --- a/libcalico-go/deps.txt +++ b/libcalico-go/deps.txt @@ -75,6 +75,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/mod v0.38.0 diff --git a/libcalico-go/lib/ipam/ipam.go b/libcalico-go/lib/ipam/ipam.go index f1018601407..7d38015dd7b 100644 --- a/libcalico-go/lib/ipam/ipam.go +++ b/libcalico-go/lib/ipam/ipam.go @@ -2406,11 +2406,20 @@ func (c ipamClient) GetUtilization(ctx context.Context, args GetUtilizationArgs) // blocks for which there is no longer an IP pool. Note: following code depends // on this being at the end of the list; otherwise it will suck in allocation // blocks that should be reported under other pools. + var orphanedBlocks *PoolUtilization if wantAllPools { - usage = append(usage, &PoolUtilization{ + orphanedBlocks = &PoolUtilization{ Name: "orphaned allocation blocks", CIDR: net.MustParseNetwork("0.0.0.0/0").IPNet, - }) + } + usage = append(usage, orphanedBlocks) + } + + // IPReservations make addresses unassignable without allocating them, so + // they have to be discounted here as well as on the allocation path. + reservations, err := c.getReservedCIDRs(ctx) + if err != nil { + return nil, err } // Read all allocation blocks. @@ -2426,15 +2435,43 @@ func (c ipamClient) GetUtilization(ctx context.Context, args GetUtilizationArgs) for _, poolUse := range usage { if b.CIDR.IsNetOverlap(poolUse.CIDR) { log.Debugf("Block CIDR %v belongs to pool %v", b.CIDR, poolUse.Name) + block := allocationBlock{b} + capacity := b.NumAddresses() poolUse.Blocks = append(poolUse.Blocks, BlockUtilization{ CIDR: b.CIDR.IPNet, - Capacity: b.NumAddresses(), - Available: len(b.Unallocated), + Capacity: capacity, + InUse: capacity - len(b.Unallocated), + Reserved: block.NumReservedAddresses(reservations), + Available: block.NumFreeAddresses(reservations), }) break } } } + + // Total up each pool. Capacity and Reserved cover the whole pool CIDR, + // including space that no block has been carved from yet, so they come from + // the pool's own arithmetic rather than from a sum over the blocks: a + // reservation over unblocked space is still unassignable. + for _, poolUse := range usage { + if poolUse == orphanedBlocks { + // Not a real pool. Its blocks are listed so that stray allocations + // stay visible, but totals over its 0.0.0.0/0 "CIDR" would be + // meaningless. + continue + } + for _, b := range poolUse.Blocks { + poolUse.InUse += b.InUse + poolUse.Available += b.Available + } + capacity, reserved, availableOutsideBlocks, err := countPoolSpace(poolUse.CIDR, reservations, poolUse.Blocks) + if err != nil { + return nil, err + } + poolUse.Capacity = capacity + poolUse.Reserved = reserved + poolUse.Available += availableOutsideBlocks + } return usage, nil } @@ -2568,32 +2605,32 @@ func (c ipamClient) ensureBlock(ctx context.Context, rsvdAttr *HostReservedAttr, } func (c ipamClient) getReservedIPs(ctx context.Context) (addrFilter, error) { - reservations, err := c.reservations.List(ctx, options.ListOptions{}) + cidrs, err := c.getReservedCIDRs(ctx) if err != nil { return nil, err } - if len(reservations.Items) == 0 { + if len(cidrs) == 0 { return nilAddrFilter{}, nil } - var cidrs cidrSliceFilter - for _, r := range reservations.Items { - for _, cidrVal := range r.Spec.ReservedCIDRs { - cidrStr := strings.TrimSpace(string(cidrVal)) - if len(cidrVal) == 0 { - // Defensive, validation should prevent. - continue - } - _, cidr, err := net.ParseCIDROrIP(cidrStr) - if err != nil { - // Defensive, validation should prevent. - log.WithError(err).WithField("cidr", cidr).Error("Ignoring malformed CIDR in IPReservation.") - } - cidrs = append(cidrs, *cidr) - } - } return cidrs, nil } +// getReservedCIDRs returns the CIDRs of every IPReservation. Callers that only +// need to test individual addresses should use getReservedIPs; the CIDRs +// themselves are for callers that need to measure how much space is reserved +// (see countPoolSpace). +func (c ipamClient) getReservedCIDRs(ctx context.Context) (cidrSliceFilter, error) { + reservations, err := c.reservations.List(ctx, options.ListOptions{}) + if err != nil { + return nil, err + } + items := make([]*v3.IPReservation, len(reservations.Items)) + for i := range reservations.Items { + items[i] = &reservations.Items[i] + } + return reservedCIDRs(items), nil +} + func (c ipamClient) UpgradeHost(ctx context.Context, nodeName string) error { delay := 100 * time.Millisecond for { diff --git a/libcalico-go/lib/ipam/ipam_block.go b/libcalico-go/lib/ipam/ipam_block.go index b191794f331..a3bee4322cc 100644 --- a/libcalico-go/lib/ipam/ipam_block.go +++ b/libcalico-go/lib/ipam/ipam_block.go @@ -238,6 +238,27 @@ func (b allocationBlock) NumFreeAddresses(reservations addrFilter) int { return len(b.Unallocated) } +// NumReservedAddresses counts the addresses in the block that the filter covers, +// whether or not they are also allocated. Where NumFreeAddresses answers "how +// many can still be handed out", this answers "how many are off limits", so the +// two overlap by any address that was allocated before it became reserved. +func (b allocationBlock) NumReservedAddresses(reservations addrFilter) int { + if reservations.MatchesWholeCIDR(&b.CIDR) { + return b.NumAddresses() + } + if !reservations.MatchesSome(&b.CIDR) { + return 0 + } + // Slow path: only some of the block is reserved, so count address by address. + numReserved := 0 + for ord := 0; ord < b.NumAddresses(); ord++ { + if reservations.MatchesIP(b.CIDR.NthIP(ord)) { + numReserved++ + } + } + return numReserved +} + // empty returns true if the block has released all of its assignable addresses, // and returns false if any assignable addresses are in use. func (b allocationBlock) empty() bool { diff --git a/libcalico-go/lib/ipam/ipam_block_test.go b/libcalico-go/lib/ipam/ipam_block_test.go index c6524504a6c..703d5c1f80d 100644 --- a/libcalico-go/lib/ipam/ipam_block_test.go +++ b/libcalico-go/lib/ipam/ipam_block_test.go @@ -161,6 +161,30 @@ var _ = Describe("Getting summary information about a block", func() { Expect(block.NumFreeAddresses(cidrSliceFilter([]cnet.IPNet{*wholeBlock}))).To(Equal(0)) }) + It("returns no reserved addresses with no reservations", func() { + block := makeTestBlock() + block.allocate([]int{10, 20}, "tens") + Expect(block.NumReservedAddresses(nilAddrFilter{})).To(Equal(0)) + }) + + It("returns reserved addresses with half the block reserved", func() { + _, secondHalf, err := cnet.ParseCIDR("100.64.0.128/25") + Expect(err).NotTo(HaveOccurred()) + block := makeTestBlock() + block.allocate([]int{10, 20, 210, 220}, "tens") + // Counts the reserved addresses whether or not they are allocated, so + // the two allocations in the reserved half are still included. + Expect(block.NumReservedAddresses(cidrSliceFilter([]cnet.IPNet{*secondHalf}))).To(Equal(128)) + }) + + It("returns reserved addresses with the entire block reserved", func() { + _, wholeBlock, err := cnet.ParseCIDR("100.64.0.128/24") + Expect(err).NotTo(HaveOccurred()) + block := makeTestBlock() + block.allocate([]int{10, 20, 210, 220}, "tens") + Expect(block.NumReservedAddresses(cidrSliceFilter([]cnet.IPNet{*wholeBlock}))).To(Equal(256)) + }) + It("correctly returns IPs by handle, whether zero, one, or many", func() { block := makeTestBlock() block.allocate([]int{3}, "singleton") diff --git a/libcalico-go/lib/ipam/ipam_test.go b/libcalico-go/lib/ipam/ipam_test.go index 1872de267f7..420efc27d23 100644 --- a/libcalico-go/lib/ipam/ipam_test.go +++ b/libcalico-go/lib/ipam/ipam_test.go @@ -2656,6 +2656,79 @@ var _ = testutils.E2eDatastoreDescribe("IPAM tests", testutils.DatastoreAll, fun }) }) + Describe("GetUtilization with reservations", func() { + host := "host-a" + + // Counts for one row of the utilization report, so that the table below + // carries the numbers and not the plumbing to fetch them. + type counts struct { + capacity, inUse, reserved, available int + } + + BeforeEach(func() { + Expect(bc.Clean()).To(Succeed()) + deleteAllPools() + applyNode(bc, kc, host, nil) + applyPool("10.0.0.0/24", true, "") + + // Assign a fixed address so that the block layout (a single + // 10.0.0.0/26) and the in-use count are the same in every case. + // Reservations are added afterwards by each entry, which is also the + // realistic order: an IP can be reserved after it was handed out. + err := ic.AssignIP(context.Background(), AssignIPArgs{ + IP: cnet.MustParseIP("10.0.0.5"), + Hostname: host, + }) + Expect(err).NotTo(HaveOccurred()) + }) + + DescribeTable("should discount reserved addresses", + func(reservedCIDRs []string, expectedPool, expectedBlock counts) { + resv := v3.NewIPReservation() + resv.Name = "resv" + resv.Spec.ReservedCIDRs = reservedCIDRs + reservations.Reservations = []v3.IPReservation{*resv} + + usage, err := ic.GetUtilization(context.Background(), GetUtilizationArgs{ + Pools: []string{"10.0.0.0/24"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(usage).To(HaveLen(1)) + Expect(usage[0].Blocks).To(HaveLen(1)) + pool, block := usage[0], usage[0].Blocks[0] + + Expect(counts{pool.Capacity, pool.InUse, pool.Reserved, pool.Available}). + To(Equal(expectedPool), "pool totals") + Expect(counts{block.Capacity, block.InUse, block.Reserved, block.Available}). + To(Equal(expectedBlock), "block totals") + }, + // A /24 pool (256 addresses) with one /26 block (64) holding a single + // allocation, 10.0.0.5. + Entry("inside the block", + []string{"10.0.0.32/30"}, + counts{capacity: 256, inUse: 1, reserved: 4, available: 251}, + counts{capacity: 64, inUse: 1, reserved: 4, available: 59}), + // The reservation covers pool space that no block has been carved + // from, so only the pool totals see it. + Entry("over pool space with no block", + []string{"10.0.0.128/25"}, + counts{capacity: 256, inUse: 1, reserved: 128, available: 127}, + counts{capacity: 64, inUse: 1, reserved: 0, available: 63}), + // In use and reserved overlap here, so they sum to more than the + // capacity; the address is only withheld from available once. + Entry("over the allocated address", + []string{"10.0.0.5/32"}, + counts{capacity: 256, inUse: 1, reserved: 1, available: 255}, + counts{capacity: 64, inUse: 1, reserved: 1, available: 63}), + // Nested and duplicated reservations must not be counted twice, and + // the block is reserved in its entirety. + Entry("overlapping each other", + []string{"10.0.0.0/25", "10.0.0.5/32", "10.0.0.64/26"}, + counts{capacity: 256, inUse: 1, reserved: 128, available: 128}, + counts{capacity: 64, inUse: 1, reserved: 64, available: 0}), + ) + }) + Describe("IPAM AutoAssign from different pools", func() { host := "host-a" pool1 := cnet.MustParseNetwork("10.0.0.0/24") @@ -2665,8 +2738,7 @@ var _ = testutils.E2eDatastoreDescribe("IPAM tests", testutils.DatastoreAll, fun findInUse := func(usage []*PoolUtilization, cidr string, expectedInUse int) bool { for _, poolUse := range usage { for _, blockUse := range poolUse.Blocks { - if (blockUse.CIDR.String() == cidr) && - (blockUse.Available == blockUse.Capacity-expectedInUse) { + if blockUse.CIDR.String() == cidr && blockUse.InUse == expectedInUse { return true } } diff --git a/libcalico-go/lib/ipam/ipam_types.go b/libcalico-go/lib/ipam/ipam_types.go index 75e5f4df4fe..6cf6c73b8a8 100644 --- a/libcalico-go/lib/ipam/ipam_types.go +++ b/libcalico-go/lib/ipam/ipam_types.go @@ -162,6 +162,11 @@ type GetUtilizationArgs struct { } // BlockUtilization reports IP utilization for a single allocation block. +// +// InUse and Reserved overlap: an IP allocated before an IPReservation covered it +// is counted in both. Available excludes both, so it is the only field that +// answers "how many IPs can still be handed out here?" and it cannot be derived +// by subtracting the other fields from Capacity. type BlockUtilization struct { // This block's CIDR. CIDR net.IPNet @@ -169,11 +174,22 @@ type BlockUtilization struct { // Number of possible IPs in this block. Capacity int - // Number of available IPs in this block. + // Number of allocated IPs in this block, whether or not they are also reserved. + InUse int + + // Number of reserved IPs in this block, whether or not they are also allocated. + Reserved int + + // Number of IPs in this block that are neither allocated nor reserved. Available int } // PoolUtilization reports IP utilization for a single IP pool. +// +// The counts cover the whole pool CIDR, including space that no allocation block +// has been carved from yet, so Capacity is not the sum of the blocks' capacities. +// InUse, Reserved and Available have the same meanings (and the same overlap) as +// in BlockUtilization. type PoolUtilization struct { // This pool's name. Name string @@ -181,6 +197,18 @@ type PoolUtilization struct { // This pool's CIDR. CIDR net.IPNet + // Number of possible IPs in this pool. + Capacity int + + // Number of allocated IPs in this pool, whether or not they are also reserved. + InUse int + + // Number of reserved IPs in this pool, whether or not they are also allocated. + Reserved int + + // Number of IPs in this pool that are neither allocated nor reserved. + Available int + // Utilization for each of this pool's blocks. Blocks []BlockUtilization } diff --git a/libcalico-go/lib/ipam/reserved.go b/libcalico-go/lib/ipam/reserved.go new file mode 100644 index 00000000000..aeece6357b1 --- /dev/null +++ b/libcalico-go/lib/ipam/reserved.go @@ -0,0 +1,166 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipam + +import ( + "fmt" + "math" + "math/big" + "net" + "net/netip" + "strings" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + "go4.org/netipx" + + cnet "github.com/projectcalico/calico/libcalico-go/lib/net" +) + +// NumReservedIPsInCIDR returns how many of the addresses in cidr the given +// IPReservations cover, whether or not they are also allocated. +// +// GetUtilization reports this alongside the allocated and free counts, but doing so +// costs a list of every allocation block. This is for callers that already hold the +// IPReservations — kube-controllers gets them from its syncer — and need only this +// number. +func NumReservedIPsInCIDR(cidr cnet.IPNet, reservations []*v3.IPReservation) (int, error) { + prefix, err := prefixFromCIDR(cidr.IPNet) + if err != nil { + return 0, err + } + + assignable, err := subtractReservations(prefix, reservedCIDRs(reservations)).IPSet() + if err != nil { + return 0, err + } + return clampToInt(new(big.Int).Sub(numIPsInPrefix(prefix), numIPsInSet(assignable))), nil +} + +// countPoolSpace reports address counts for a whole IP pool CIDR, including the +// parts of it that no allocation block covers yet: +// +// - capacity: how many IPs the pool CIDR holds; +// - reserved: how many of those a reservation covers, whether or not they are +// also allocated; +// - availableOutsideBlocks: how many are neither reserved nor inside one of +// the given blocks. IPs inside a block are left to the caller, which has +// the block's allocations and so can tell free from in-use. +func countPoolSpace(poolCIDR net.IPNet, reserved cidrSliceFilter, blocks []BlockUtilization) (capacity, reservedCount, availableOutsideBlocks int, err error) { + poolPrefix, err := prefixFromCIDR(poolCIDR) + if err != nil { + return 0, 0, 0, err + } + assignable := subtractReservations(poolPrefix, reserved) + + // Clone before subtracting the blocks so we can measure the pool both with + // and without them. Clone drops errors accumulated so far, but they stay on + // the original, which we check below. + outsideBlocks := assignable.Clone() + for _, b := range blocks { + if p, ok := netipx.FromStdIPNet(&b.CIDR); ok { + outsideBlocks.RemovePrefix(p) + } + } + + assignableSet, err := assignable.IPSet() + if err != nil { + return 0, 0, 0, err + } + outsideBlocksSet, err := outsideBlocks.IPSet() + if err != nil { + return 0, 0, 0, err + } + + poolSize := numIPsInPrefix(poolPrefix) + return clampToInt(poolSize), + clampToInt(new(big.Int).Sub(poolSize, numIPsInSet(assignableSet))), + clampToInt(numIPsInSet(outsideBlocksSet)), + nil +} + +// reservedCIDRs returns the CIDRs that the given IPReservations cover. Malformed +// entries are logged and skipped; validation should prevent them. +func reservedCIDRs(reservations []*v3.IPReservation) cidrSliceFilter { + var cidrs cidrSliceFilter + for _, r := range reservations { + for _, cidrStr := range r.Spec.ReservedCIDRs { + cidrStr = strings.TrimSpace(cidrStr) + if cidrStr == "" { + continue + } + _, cidr, err := cnet.ParseCIDROrIP(cidrStr) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "reservation": r.Name, + "cidr": cidrStr, + }).Error("Ignoring malformed CIDR in IPReservation.") + continue + } + cidrs = append(cidrs, *cidr) + } + } + return cidrs +} + +// subtractReservations returns the part of prefix that no reservation covers. +// +// Reservations may overlap and nest arbitrarily — one IPReservation can cover a /24 +// while another names a single address inside it — so this has to be a set operation +// rather than a sum over the CIDRs. Subtracting a prefix splits whatever it partly +// overlaps and repeats are no-ops, so the set needs no deduplication of our own. +func subtractReservations(prefix netip.Prefix, reserved cidrSliceFilter) *netipx.IPSetBuilder { + var assignable netipx.IPSetBuilder + assignable.AddPrefix(prefix) + for _, r := range reserved { + if p, ok := netipx.FromStdIPNet(&r.IPNet); ok { + assignable.RemovePrefix(p) + } else { + log.WithField("cidr", r.String()).Warn("Ignoring reservation that cannot be represented as a prefix.") + } + } + return &assignable +} + +func prefixFromCIDR(cidr net.IPNet) (netip.Prefix, error) { + p, ok := netipx.FromStdIPNet(&cidr) + if !ok { + return netip.Prefix{}, fmt.Errorf("CIDR %s cannot be represented as a prefix", cidr.String()) + } + return p, nil +} + +func numIPsInSet(s *netipx.IPSet) *big.Int { + total := big.NewInt(0) + for _, p := range s.Prefixes() { + total.Add(total, numIPsInPrefix(p)) + } + return total +} + +func numIPsInPrefix(p netip.Prefix) *big.Int { + return new(big.Int).Lsh(big.NewInt(1), uint(p.Addr().BitLen()-p.Bits())) +} + +// clampToInt saturates rather than wrapping. Real pools are far smaller than +// this — IPv6 pools are /96 or longer — so it is purely defensive: it keeps the +// conversion to the int fields of BlockUtilization and PoolUtilization total, +// instead of turning an oversized count negative. +func clampToInt(n *big.Int) int { + if !n.IsInt64() || n.Int64() > math.MaxInt { + return math.MaxInt + } + return int(n.Int64()) +} diff --git a/libcalico-go/lib/ipam/reserved_test.go b/libcalico-go/lib/ipam/reserved_test.go new file mode 100644 index 00000000000..3c98bae0908 --- /dev/null +++ b/libcalico-go/lib/ipam/reserved_test.go @@ -0,0 +1,211 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipam + +import ( + "fmt" + "math" + "slices" + "testing" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + cnet "github.com/projectcalico/calico/libcalico-go/lib/net" +) + +func TestCountPoolSpace(t *testing.T) { + for _, tc := range []struct { + name string + pool string + reservations []string + blocks []string + wantCapacity int + wantReserved int + wantAvailableOutsideBlock int + }{ + { + name: "no reservations and no blocks", + pool: "10.0.0.0/24", + wantCapacity: 256, + wantAvailableOutsideBlock: 256, + }, + { + name: "one reservation", + pool: "10.0.0.0/24", + reservations: []string{"10.0.0.32/30"}, + wantCapacity: 256, + wantReserved: 4, + wantAvailableOutsideBlock: 252, + }, + { + // The nested and duplicated CIDRs must not be counted more than once. + name: "overlapping reservations", + pool: "10.0.0.0/24", + reservations: []string{"10.0.0.0/25", "10.0.0.5/32", "10.0.0.64/26", "10.0.0.0/25"}, + wantCapacity: 256, + wantReserved: 128, + wantAvailableOutsideBlock: 128, + }, + { + name: "reservation covering the whole pool", + pool: "10.0.0.0/24", + reservations: []string{"10.0.0.0/16"}, + wantCapacity: 256, + wantReserved: 256, + }, + { + name: "reservation outside the pool", + pool: "10.0.0.0/24", + reservations: []string{"192.168.0.0/24", "fd00::/120"}, + wantCapacity: 256, + wantAvailableOutsideBlock: 256, + }, + { + name: "blocks carved from the pool", + pool: "10.0.0.0/24", + blocks: []string{"10.0.0.0/26", "10.0.0.64/26"}, + wantCapacity: 256, + wantAvailableOutsideBlock: 128, + }, + { + // The reservation is inside a block, so it does not reduce the space + // outside the blocks; the block's own count covers it. + name: "reservation inside a block", + pool: "10.0.0.0/24", + reservations: []string{"10.0.0.32/30"}, + blocks: []string{"10.0.0.0/26"}, + wantCapacity: 256, + wantReserved: 4, + wantAvailableOutsideBlock: 192, + }, + { + name: "reservation outside every block", + pool: "10.0.0.0/24", + reservations: []string{"10.0.0.128/25"}, + blocks: []string{"10.0.0.0/26"}, + wantCapacity: 256, + wantReserved: 128, + wantAvailableOutsideBlock: 64, + }, + { + name: "IPv6 pool", + pool: "fd00::/120", + reservations: []string{"fd00::/126"}, + wantCapacity: 256, + wantReserved: 4, + wantAvailableOutsideBlock: 252, + }, + { + // Bigger than validation allows, but the counts must saturate rather + // than wrap if one ever gets this far. + name: "pool too big for an int", + pool: "fd00::/8", + wantCapacity: math.MaxInt, + wantAvailableOutsideBlock: math.MaxInt, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var reservations cidrSliceFilter + for _, r := range tc.reservations { + reservations = append(reservations, cnet.MustParseNetwork(r)) + } + var blocks []BlockUtilization + for _, b := range tc.blocks { + blocks = append(blocks, BlockUtilization{CIDR: cnet.MustParseNetwork(b).IPNet}) + } + + capacity, reserved, availableOutsideBlocks, err := countPoolSpace( + cnet.MustParseNetwork(tc.pool).IPNet, reservations, blocks) + if err != nil { + t.Fatalf("countPoolSpace returned an error: %v", err) + } + if capacity != tc.wantCapacity { + t.Errorf("capacity = %d, want %d", capacity, tc.wantCapacity) + } + if reserved != tc.wantReserved { + t.Errorf("reserved = %d, want %d", reserved, tc.wantReserved) + } + if availableOutsideBlocks != tc.wantAvailableOutsideBlock { + t.Errorf("availableOutsideBlocks = %d, want %d", availableOutsideBlocks, tc.wantAvailableOutsideBlock) + } + + // The exported entry point skips the block arithmetic, but its reserved + // count must agree with the one above; kube-controllers reports that + // number for the same pools that calicoctl shows. + numReserved, err := NumReservedIPsInCIDR(cnet.MustParseNetwork(tc.pool), reservationsCovering(tc.reservations)) + if err != nil { + t.Fatalf("NumReservedIPsInCIDR returned an error: %v", err) + } + if numReserved != tc.wantReserved { + t.Errorf("NumReservedIPsInCIDR = %d, want %d", numReserved, tc.wantReserved) + } + }) + } +} + +// reservationsCovering returns one IPReservation per CIDR, which is the interesting +// shape: reservations that overlap each other arrive as separate resources. +func reservationsCovering(cidrs []string) []*v3.IPReservation { + var reservations []*v3.IPReservation + for i, cidr := range cidrs { + reservations = append(reservations, &v3.IPReservation{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("reservation-%d", i)}, + Spec: v3.IPReservationSpec{ReservedCIDRs: []string{cidr}}, + }) + } + return reservations +} + +func TestReservedCIDRs(t *testing.T) { + for _, tc := range []struct { + name string + reserved []string + want []string + }{ + { + name: "CIDRs and bare IPs", + reserved: []string{"10.0.0.0/24", "10.1.0.1", "fd00::1"}, + want: []string{"10.0.0.0/24", "10.1.0.1/32", "fd00::1/128"}, + }, + { + name: "surrounding whitespace", + reserved: []string{" 10.0.0.0/24 "}, + want: []string{"10.0.0.0/24"}, + }, + { + // Validation should prevent all of these, but a hand-written CRD can + // still carry them and they must not take the count with them. + name: "malformed entries are skipped", + reserved: []string{"", " ", "not-a-cidr", "10.0.0.0/33", "10.0.0.0/24"}, + want: []string{"10.0.0.0/24"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cidrs := reservedCIDRs([]*v3.IPReservation{{ + ObjectMeta: metav1.ObjectMeta{Name: "reservation"}, + Spec: v3.IPReservationSpec{ReservedCIDRs: tc.reserved}, + }}) + + var got []string + for _, c := range cidrs { + got = append(got, c.String()) + } + if !slices.Equal(got, tc.want) { + t.Errorf("reservedCIDRs = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 5d4e37ec036..0fb8e4c63be 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -11957,6 +11957,7 @@ rules: - ipreservations verbs: - list + - watch - apiGroups: ["projectcalico.org", "crd.projectcalico.org"] resources: - blockaffinities diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index 2a51fd92201..2338567d8de 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -11967,6 +11967,7 @@ rules: - ipreservations verbs: - list + - watch - apiGroups: ["projectcalico.org", "crd.projectcalico.org"] resources: - blockaffinities diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 518d1b3b602..40befcb747c 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -11968,6 +11968,7 @@ rules: - ipreservations verbs: - list + - watch - apiGroups: ["projectcalico.org", "crd.projectcalico.org"] resources: - blockaffinities diff --git a/manifests/calico-v3-crds.yaml b/manifests/calico-v3-crds.yaml index cdb4f4ec1ac..ae65eb510e7 100644 --- a/manifests/calico-v3-crds.yaml +++ b/manifests/calico-v3-crds.yaml @@ -12309,6 +12309,7 @@ rules: - ipreservations verbs: - list + - watch - apiGroups: ["projectcalico.org", "crd.projectcalico.org"] resources: - blockaffinities diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index dbeb951b890..9f1ddbddc3a 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -11952,6 +11952,7 @@ rules: - ipreservations verbs: - list + - watch - apiGroups: ["projectcalico.org", "crd.projectcalico.org"] resources: - blockaffinities diff --git a/manifests/calico.yaml b/manifests/calico.yaml index 06666e73100..1cae27369f1 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -11952,6 +11952,7 @@ rules: - ipreservations verbs: - list + - watch - apiGroups: ["projectcalico.org", "crd.projectcalico.org"] resources: - blockaffinities diff --git a/manifests/canal.yaml b/manifests/canal.yaml index c4a95d471cd..7ec4079d69a 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -11969,6 +11969,7 @@ rules: - ipreservations verbs: - list + - watch - apiGroups: ["projectcalico.org", "crd.projectcalico.org"] resources: - blockaffinities diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index 3ffab71eaa6..8c6fdaf0ed3 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -11952,6 +11952,7 @@ rules: - ipreservations verbs: - list + - watch - apiGroups: ["projectcalico.org", "crd.projectcalico.org"] resources: - blockaffinities diff --git a/node/deps.txt b/node/deps.txt index 525d9349f44..b4f687e02da 100644 --- a/node/deps.txt +++ b/node/deps.txt @@ -130,6 +130,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/net v0.57.0 diff --git a/typha/deps.txt b/typha/deps.txt index c7f11e8a228..7778a2e68d2 100644 --- a/typha/deps.txt +++ b/typha/deps.txt @@ -67,6 +67,7 @@ go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v2 v2.4.4 go.yaml.in/yaml/v3 v3.0.4 +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba golang.org/x/crypto v0.54.0 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 golang.org/x/net v0.57.0