diff --git a/src/reconcile/tls.rs b/src/reconcile/tls.rs index e2b384e..fc1f2fc 100644 --- a/src/reconcile/tls.rs +++ b/src/reconcile/tls.rs @@ -30,7 +30,7 @@ use k8s_openapi::api::core::v1::Secret; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use kube::api::{Api, DeleteParams, ListParams, Patch, PatchParams}; use kube::core::{ApiResource, DynamicObject, GroupVersionKind}; -use rustls::pki_types::{CertificateDer, DnsName, ServerName}; +use rustls::pki_types::{DnsName, ServerName}; use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; use std::collections::BTreeSet; @@ -52,6 +52,32 @@ const CERT_MANAGER_ISSUER_PLURAL: &str = "issuers"; const CERT_MANAGER_CLUSTER_ISSUER_KIND: &str = "ClusterIssuer"; const CERT_MANAGER_CLUSTER_ISSUER_PLURAL: &str = "clusterissuers"; const STATUS_MESSAGE_LIMIT: usize = 256; +// etcd's default 1.5 MiB request limit is the hard upper envelope for Kubernetes objects. Keep +// explicit space for the etcd key, protobuf transaction envelope, admission defaults, and other +// server-side metadata that are not represented by the JSON payload serialized here. +const ETCD_DEFAULT_MAX_REQUEST_BYTES: u64 = 3 * 1024 * 1024 / 2; +const KUBERNETES_OBJECT_PROTOCOL_MARGIN_BYTES: u64 = 64 * 1024; +const MAX_KUBERNETES_OBJECT_JSON_BYTES: u64 = + ETCD_DEFAULT_MAX_REQUEST_BYTES - KUBERNETES_OBJECT_PROTOCOL_MARGIN_BYTES; +// These are allocation caps, not substitutes for the complete-object checks below. +const MAX_CERTIFICATE_SAN_BYTES: u64 = MAX_KUBERNETES_OBJECT_JSON_BYTES; +const MAX_TLS_STATUS_SAN_BYTES: u64 = 512 * 1024; +const TLS_STATUS_FIXED_ESTIMATE_BYTES: u64 = 16 * 1024; +const TLS_STATUS_PER_CERTIFICATE_ESTIMATE_BYTES: u64 = 2 * 1024; +// A String plus BTreeSet/Vec bookkeeping is conservatively budgeted at 64 bytes independent of +// the string payload, which is limited separately by serialized byte budgets. +const SAN_NAME_CONTAINER_OVERHEAD_BYTES: u64 = 64; +const MAX_TLS_STATUS_SAN_NAME_COPIES: u64 = + MAX_TLS_STATUS_SAN_BYTES / SAN_NAME_CONTAINER_OVERHEAD_BYTES; +const MAX_TLS_SAN_NAMES_PER_LIST: u64 = MAX_TLS_STATUS_SAN_NAME_COPIES; +const MAX_TLS_SAN_COVERAGE_LOOKUPS: u64 = MAX_TLS_STATUS_SAN_NAME_COPIES * 2; +const MAX_TLS_CERTIFICATE_ENTRIES: u64 = 64; +const MAX_TLS_SECRET_MATERIAL_BYTES: u64 = 16 * 1024 * 1024; +const MAX_TLS_CERTIFICATE_DER_BYTES: u64 = 8 * 1024 * 1024; +// Concrete-name verification is delegated to webpki, which may scan the complete certificate SAN +// extension for each name. This is a single reconcile-wide budget, not a per-certificate limit. +const MAX_TLS_SAN_VERIFICATION_SCAN_BYTES: u64 = 16 * 1024 * 1024; +const STATUS_DNS_NAMES_FIELD_OVERHEAD_BYTES: u64 = 16; #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum CertManagerPrerequisite { @@ -94,11 +120,112 @@ struct TlsCertificateEntry { struct ObservedTlsCertificate { entry: TlsCertificateEntry, secret_name: String, - secret: Secret, + secret_resource_version: Option, + cert_bytes: Vec, + ca_bytes: Option>, certificate_ref: Option, san_dns_names: Vec, } +#[derive(Debug, Default)] +struct TlsCertificateRuntimeBudget { + secret_material_bytes: u64, + der_bytes: u64, + verification_scan_bytes: u64, +} + +impl TlsCertificateRuntimeBudget { + fn charge_secret_material(&mut self, bytes: usize) -> Result<(), TlsValidationFailure> { + self.secret_material_bytes = checked_resource_add( + self.secret_material_bytes, + bytes, + "PEM/secret-material byte", + )?; + if self.secret_material_bytes > MAX_TLS_SECRET_MATERIAL_BYTES { + return Err(resource_budget_failure(format!( + "TLS reconciliation loaded {} PEM/secret-material bytes, exceeding the reconcile-wide budget of {MAX_TLS_SECRET_MATERIAL_BYTES}", + self.secret_material_bytes + ))); + } + Ok(()) + } + + fn charge_der(&mut self, der_bytes: usize) -> Result { + let der_bytes = u64::try_from(der_bytes).map_err(|_| { + resource_budget_failure( + "TLS certificate DER length overflowed its resource budget".to_string(), + ) + })?; + self.der_bytes = self.der_bytes.checked_add(der_bytes).ok_or_else(|| { + resource_budget_failure( + "TLS certificate DER byte count overflowed its resource budget".to_string(), + ) + })?; + if self.der_bytes > MAX_TLS_CERTIFICATE_DER_BYTES { + return Err(resource_budget_failure(format!( + "TLS reconciliation parsed {} certificate DER bytes, exceeding the reconcile-wide budget of {MAX_TLS_CERTIFICATE_DER_BYTES}", + self.der_bytes + ))); + } + Ok(der_bytes) + } + + fn charge_scan( + &mut self, + der_bytes: u64, + concrete_names: u64, + ) -> Result<(), TlsValidationFailure> { + // valid_dns_names performs one complete SAN traversal before webpki performs one traversal + // for every concrete reference name. + let scans = concrete_names.checked_add(1).ok_or_else(|| { + resource_budget_failure( + "TLS SAN verification scan count overflowed its resource budget".to_string(), + ) + })?; + let scan_bytes = der_bytes.checked_mul(scans).ok_or_else(|| { + resource_budget_failure( + "TLS SAN verification work calculation overflowed its resource budget".to_string(), + ) + })?; + self.verification_scan_bytes = self + .verification_scan_bytes + .checked_add(scan_bytes) + .ok_or_else(|| { + resource_budget_failure( + "TLS SAN verification work calculation overflowed its resource budget" + .to_string(), + ) + })?; + if self.verification_scan_bytes > MAX_TLS_SAN_VERIFICATION_SCAN_BYTES { + return Err(resource_budget_failure(format!( + "TLS SAN validation may scan {} certificate DER bytes, exceeding the reconcile-wide verification budget of {MAX_TLS_SAN_VERIFICATION_SCAN_BYTES}", + self.verification_scan_bytes + ))); + } + Ok(()) + } +} + +fn checked_resource_add( + current: u64, + bytes: usize, + label: &str, +) -> Result { + let bytes = u64::try_from(bytes).map_err(|_| { + resource_budget_failure(format!("TLS {label} count overflowed its resource budget")) + })?; + current.checked_add(bytes).ok_or_else(|| { + resource_budget_failure(format!("TLS {label} count overflowed its resource budget")) + }) +} + +fn resource_budget_failure(message: String) -> TlsValidationFailure { + TlsValidationFailure { + reason: Reason::CertificateInvalid, + message, + } +} + impl CertManagerCertificateObservation { fn status_ref(&self) -> CertificateObjectRef { CertificateObjectRef { @@ -176,6 +303,14 @@ async fn reconcile_cert_manager_tls( Err(failure) => return tls_validation_blocked(ctx, tenant, config, failure).await, }; + let cluster_domain = ctx.cluster_domain(); + let san_budget = match san_budget::validate(tenant, namespace, cluster_domain, config, &entries) + { + Ok(budget) => budget, + Err(failure) => return tls_validation_blocked(ctx, tenant, config, failure).await, + }; + let mut runtime_budget = TlsCertificateRuntimeBudget::default(); + if entries .iter() .any(|entry| entry.cert_manager.manage_certificate) @@ -197,7 +332,6 @@ async fn reconcile_cert_manager_tls( let mut observed = Vec::with_capacity(entries.len()); let mut desired_managed_certificate_names = BTreeSet::new(); - let cluster_domain = ctx.cluster_domain(); for entry in entries { let cert_manager = &entry.cert_manager; let Some(secret_name) = cert_manager @@ -237,13 +371,9 @@ async fn reconcile_cert_manager_tls( let certificate_name = certificate_name(tenant, &entry); desired_managed_certificate_names.insert(certificate_name.clone()); - if let Err(failure) = validate_managed_certificate_san_config( - tenant, - namespace, - config, - &entry, - cluster_domain, - ) { + if let Err(failure) = + validate_managed_certificate_san_config(&san_budget, config, &entry) + { return tls_validation_blocked(ctx, tenant, config, failure).await; } @@ -263,17 +393,20 @@ async fn reconcile_cert_manager_tls( } let desired_certificate = build_cert_manager_certificate( - tenant, - namespace, + &san_budget, cert_manager, &entry.hosts, include_generated_dns_names(&entry), - cluster_domain, CertManagerCertificateNames { secret: &secret_name, certificate: &certificate_name, }, ); + if let Err(failure) = + san_budget::validate_certificate_object(&desired_certificate, &certificate_name) + { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } let observed_certificate = match apply_cert_manager_certificate( ctx, namespace, @@ -358,22 +491,42 @@ async fn reconcile_cert_manager_tls( Reason::CertificateSecretMissingKey, ) .await?; - require_secret_key( - ctx, - tenant, - config, + if let Err(failure) = runtime_budget.charge_secret_material(cert_bytes.len()) { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } + let key_bytes = match require_secret_key_bytes( &secret, &secret_name, TLS_KEY_KEY, Reason::CertificateSecretMissingKey, - ) - .await?; + ) { + Ok(bytes) => bytes, + Err(failure) => return tls_validation_blocked(ctx, tenant, config, failure).await, + }; + if let Err(failure) = runtime_budget.charge_secret_material(key_bytes.len()) { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } + + let ca_bytes = entry + .default + .then(|| secret_bytes(&secret, CA_CERT_KEY)) + .flatten() + .map(ToOwned::to_owned); + if let Some(bytes) = ca_bytes.as_deref() + && let Err(failure) = runtime_budget.charge_secret_material(bytes.len()) + { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } + let secret_resource_version = secret.metadata.resource_version.clone(); - let san_dns_names = - san_validation_dns_names(tenant, namespace, config, &entry, cluster_domain); + let san_dns_names = san_validation_dns_names(&san_budget, config, &entry); if config.require_san_match - && let Err(failure) = - validate_tls_secret_san_match(&secret_name, &cert_bytes, &san_dns_names) + && let Err(failure) = validate_tls_secret_san_match_with_budget( + &secret_name, + &cert_bytes, + &san_dns_names, + &mut runtime_budget, + ) { return tls_validation_blocked(ctx, tenant, config, failure).await; } @@ -381,7 +534,9 @@ async fn reconcile_cert_manager_tls( observed.push(ObservedTlsCertificate { entry, secret_name, - secret, + secret_resource_version, + cert_bytes, + ca_bytes, certificate_ref, san_dns_names, }); @@ -409,11 +564,12 @@ async fn reconcile_cert_manager_tls( let mut explicit_ca_bytes: Option> = None; match ca_trust.source { - CaTrustSource::CertificateSecretCa => match certificate_secret_ca_material( - &default_certificate.secret, + CaTrustSource::CertificateSecretCa => match certificate_ca_material( + default_certificate.ca_bytes.as_deref(), &default_certificate.secret_name, config.enable_internode_https, trust_system_ca, + &mut runtime_budget, ) { Ok(Some(material)) => { server_ca_key = Some(material.key); @@ -453,10 +609,14 @@ async fn reconcile_cert_manager_tls( Reason::CaBundleMissing, ) .await?; + if let Err(failure) = runtime_budget.charge_secret_material(ca_bytes.len()) { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } if let Err(failure) = validate_ca_bundle_bytes( &ca_secret_ref.name, &ca_secret_ref.key, ca_bytes.as_slice(), + &mut runtime_budget, ) { return tls_validation_blocked(ctx, tenant, config, failure).await; } @@ -496,10 +656,16 @@ async fn reconcile_cert_manager_tls( ) .await?, ); + if let Some(bytes) = client_ca_bytes.as_deref() + && let Err(failure) = runtime_budget.charge_secret_material(bytes.len()) + { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } if let Err(failure) = validate_ca_bundle_bytes( &client_ca_secret_ref.name, &client_ca_secret_ref.key, client_ca_bytes.as_deref().unwrap_or_default(), + &mut runtime_budget, ) { return tls_validation_blocked(ctx, tenant, config, failure).await; } @@ -544,6 +710,9 @@ async fn reconcile_cert_manager_tls( client_ca.as_ref().zip(client_ca_secret.as_ref()), &hash, ); + if let Err(failure) = san_budget::validate_tenant_tls_status(tenant, &status) { + return tls_validation_blocked(ctx, tenant, config, failure).await; + } let server_certificates = observed .iter() .map(|certificate| TlsServerCertificateMount { @@ -902,14 +1071,14 @@ struct CertManagerCertificateNames<'a> { } fn build_cert_manager_certificate( - tenant: &Tenant, - namespace: &str, + san_budget: &ValidatedTlsSanBudget<'_>, cert_manager: &CertManagerTlsConfig, hosts: &[String], include_generated_dns_names: bool, - cluster_domain: &str, names: CertManagerCertificateNames<'_>, ) -> DynamicObject { + let tenant = san_budget.tenant(); + let namespace = san_budget.namespace(); let mut spec = Map::new(); spec.insert("secretName".to_string(), json!(names.secret)); if let Some(issuer_ref) = cert_manager.issuer_ref.as_ref() { @@ -925,12 +1094,10 @@ fn build_cert_manager_certificate( spec.insert( "dnsNames".to_string(), json!(certificate_dns_names( - tenant, - namespace, + san_budget, cert_manager, hosts, include_generated_dns_names, - cluster_domain, )), ); spec.insert( @@ -1117,13 +1284,14 @@ fn issuer_ref_value(issuer_ref: &CertManagerIssuerRef) -> Value { } fn certificate_dns_names( - tenant: &Tenant, - namespace: &str, + san_budget: &ValidatedTlsSanBudget<'_>, cert_manager: &CertManagerTlsConfig, hosts: &[String], include_generated_dns_names: bool, - cluster_domain: &str, ) -> Vec { + let tenant = san_budget.tenant(); + let namespace = san_budget.namespace(); + let cluster_domain = san_budget.cluster_domain(); let mut names = BTreeSet::new(); names.extend(hosts.iter().filter(|name| !name.is_empty()).cloned()); names.extend( @@ -1171,57 +1339,681 @@ fn include_generated_dns_names(entry: &TlsCertificateEntry) -> bool { .unwrap_or(entry.default || entry.legacy) } +mod san_budget { + use super::*; + + #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] + struct SanListEstimate { + names: u64, + encoded_item_bytes: u64, + } + + impl SanListEstimate { + fn add_name(&mut self, name: &str) -> Result<(), TlsValidationFailure> { + let encoded = serde_json::to_string(name).map_err(|_| overflow_failure())?; + self.names = checked_add(self.names, 1)?; + self.encoded_item_bytes = checked_add( + self.encoded_item_bytes, + checked_add(to_u64(encoded.len())?, 1)?, + )?; + Ok(()) + } + + fn add_ascii_batch( + &mut self, + count: u64, + total_name_bytes: u64, + ) -> Result<(), TlsValidationFailure> { + self.names = checked_add(self.names, count)?; + let quotes_and_commas = checked_multiply(count, 3)?; + self.encoded_item_bytes = checked_add( + self.encoded_item_bytes, + checked_add(total_name_bytes, quotes_and_commas)?, + )?; + Ok(()) + } + + fn merge(&mut self, other: Self) -> Result<(), TlsValidationFailure> { + self.names = checked_add(self.names, other.names)?; + self.encoded_item_bytes = + checked_add(self.encoded_item_bytes, other.encoded_item_bytes)?; + Ok(()) + } + + fn array_bytes(self) -> Result { + checked_add(self.encoded_item_bytes, 2) + } + } + + /// Proof that every `servers`-driven SAN expansion for this Tenant, namespace, and cluster + /// domain fits the Certificate, status, allocation, and comparison budgets. The private proof + /// field prevents callers outside this module from constructing a token without validation. + #[derive(Debug)] + pub(super) struct ValidatedTlsSanBudget<'a> { + tenant: &'a Tenant, + namespace: &'a str, + cluster_domain: &'a str, + _proof: Proof, + } + + #[derive(Debug)] + struct Proof; + + impl<'a> ValidatedTlsSanBudget<'a> { + pub(super) fn tenant(&self) -> &'a Tenant { + self.tenant + } + + pub(super) fn namespace(&self) -> &'a str { + self.namespace + } + + pub(super) fn cluster_domain(&self) -> &'a str { + self.cluster_domain + } + } + + pub(super) fn validate<'a>( + tenant: &'a Tenant, + namespace: &'a str, + cluster_domain: &'a str, + config: &TlsConfig, + entries: &[TlsCertificateEntry], + ) -> Result, TlsValidationFailure> { + if config.mode != TlsMode::CertManager { + return Ok(token(tenant, namespace, cluster_domain)); + } + let entry_count = to_u64(entries.len())?; + if entry_count > MAX_TLS_CERTIFICATE_ENTRIES { + return Err(budget_failure(format!( + "spec.tls contains {entry_count} certificate entries, exceeding the reconcile-wide limit of {MAX_TLS_CERTIFICATE_ENTRIES}" + ))); + } + + let generated = generated_names_estimate(tenant, namespace, cluster_domain)?; + let required = required_names_estimate(tenant, namespace, cluster_domain)?; + let mut status_bytes = 0_u64; + let mut status_name_copies = 0_u64; + let mut coverage_lookups = 0_u64; + + for entry in entries { + let custom = custom_names_estimate(entry)?; + let status_hosts = status_hosts_estimate(entry)?; + let include_generated = include_generated_dns_names(entry); + + if entry.cert_manager.manage_certificate { + let mut certificate = custom; + if include_generated { + certificate.merge(generated)?; + } + validate_list( + &format!("cert-manager Certificate '{}' dnsNames", entry.name), + certificate, + MAX_CERTIFICATE_SAN_BYTES, + )?; + } + + let mut status = custom; + if config.enable_internode_https && entry.default { + status.merge(required)?; + } else if include_generated { + status.merge(generated)?; + } + validate_list( + &format!("TLS status for certificate '{}'", entry.name), + status, + MAX_TLS_STATUS_SAN_BYTES, + )?; + add_status_copy(&mut status_bytes, &mut status_name_copies, status)?; + // certificates[].hosts is serialized independently from certificates[].dnsNames. + // Hosts are also part of dnsNames, so both copies must be charged to the status budget. + add_status_copy(&mut status_bytes, &mut status_name_copies, status_hosts)?; + if entry.default { + // The default certificate SAN list is retained both in certificates[] and in the + // compatibility top-level status.dnsNames field. + add_status_copy(&mut status_bytes, &mut status_name_copies, status)?; + } + + if entry.cert_manager.manage_certificate + && config.enable_internode_https + && entry.default + && !include_generated + { + coverage_lookups = checked_add(coverage_lookups, custom.names)?; + coverage_lookups = checked_add(coverage_lookups, required.names)?; + } + if config.require_san_match { + coverage_lookups = checked_add(coverage_lookups, status.names)?; + } + } + + if status_bytes > MAX_TLS_STATUS_SAN_BYTES { + return Err(budget_failure(format!( + "spec.tls would serialize at least {status_bytes} bytes of DNS SAN lists into Tenant status, exceeding the {MAX_TLS_STATUS_SAN_BYTES}-byte TLS status budget derived from etcd's {ETCD_DEFAULT_MAX_REQUEST_BYTES}-byte request limit" + ))); + } + if status_name_copies > MAX_TLS_STATUS_SAN_NAME_COPIES { + return Err(budget_failure(format!( + "spec.tls would retain {status_name_copies} DNS SAN name copies in Tenant status, exceeding the allocation budget of {MAX_TLS_STATUS_SAN_NAME_COPIES}" + ))); + } + if coverage_lookups > MAX_TLS_SAN_COVERAGE_LOOKUPS { + return Err(budget_failure(format!( + "spec.tls requires {coverage_lookups} DNS SAN coverage operations, exceeding the comparison budget of {MAX_TLS_SAN_COVERAGE_LOOKUPS}" + ))); + } + validate_projected_tenant_status_size(tenant, config, entry_count, status_bytes)?; + + Ok(token(tenant, namespace, cluster_domain)) + } + + pub(super) fn validate_certificate_object( + certificate: &DynamicObject, + name: &str, + ) -> Result<(), TlsValidationFailure> { + validate_complete_object_size(certificate, &format!("cert-manager Certificate '{name}'")) + } + + pub(super) fn validate_tenant_tls_status( + tenant: &Tenant, + tls_status: &TlsCertificateStatus, + ) -> Result<(), TlsValidationFailure> { + let mut candidate = tenant.clone(); + let status = candidate.status.get_or_insert_default(); + status.certificates.tls = Some(tls_status.clone()); + validate_complete_object_size(&candidate, "Tenant with reconciled TLS status") + } + + fn token<'a>( + tenant: &'a Tenant, + namespace: &'a str, + cluster_domain: &'a str, + ) -> ValidatedTlsSanBudget<'a> { + ValidatedTlsSanBudget { + tenant, + namespace, + cluster_domain, + _proof: Proof, + } + } + + fn validate_list( + label: &str, + estimate: SanListEstimate, + max_bytes: u64, + ) -> Result<(), TlsValidationFailure> { + if estimate.names > MAX_TLS_SAN_NAMES_PER_LIST { + return Err(budget_failure(format!( + "{label} would contain {} names, exceeding the per-list allocation budget of {MAX_TLS_SAN_NAMES_PER_LIST}", + estimate.names + ))); + } + let bytes = estimate.array_bytes()?; + if bytes > max_bytes { + return Err(budget_failure(format!( + "{label} would serialize to at least {bytes} bytes, exceeding its {max_bytes}-byte budget derived from etcd's {ETCD_DEFAULT_MAX_REQUEST_BYTES}-byte request limit" + ))); + } + Ok(()) + } + + fn add_status_copy( + total_bytes: &mut u64, + total_names: &mut u64, + estimate: SanListEstimate, + ) -> Result<(), TlsValidationFailure> { + if estimate.names == 0 { + return Ok(()); + } + *total_bytes = checked_add( + *total_bytes, + checked_add( + estimate.array_bytes()?, + STATUS_DNS_NAMES_FIELD_OVERHEAD_BYTES, + )?, + )?; + *total_names = checked_add(*total_names, estimate.names)?; + Ok(()) + } + + fn validate_projected_tenant_status_size( + tenant: &Tenant, + config: &TlsConfig, + entry_count: u64, + status_san_bytes: u64, + ) -> Result<(), TlsValidationFailure> { + let mut tenant_without_tls = tenant.clone(); + if let Some(status) = tenant_without_tls.status.as_mut() { + status.certificates.tls = None; + } + let base_bytes = serialized_len(&tenant_without_tls)?; + let config_bytes = serialized_len(config)?; + // The ready status repeats only a subset of TLS config, but charge two complete config + // copies so arbitrary user strings are never hidden behind a fixed "rest of object" + // allowance. Per-certificate and fixed allowances cover observed resourceVersions, + // Certificate condition metadata, timestamps, hashes, JSON field names, and future small + // status additions. The protocol margin is checked separately for the complete object. + let repeated_config_bytes = checked_multiply(config_bytes, 2)?; + let certificate_metadata_bytes = + checked_multiply(entry_count, TLS_STATUS_PER_CERTIFICATE_ESTIMATE_BYTES)?; + let projected_status_bytes = checked_add( + checked_add(status_san_bytes, repeated_config_bytes)?, + checked_add(TLS_STATUS_FIXED_ESTIMATE_BYTES, certificate_metadata_bytes)?, + )?; + let projected_object_bytes = checked_add(base_bytes, projected_status_bytes)?; + validate_object_bytes(projected_object_bytes, "Tenant with projected TLS status") + } + + fn validate_complete_object_size( + value: &T, + label: &str, + ) -> Result<(), TlsValidationFailure> { + validate_object_bytes(serialized_len(value)?, label) + } + + fn serialized_len(value: &T) -> Result { + let bytes = serde_json::to_vec(value).map_err(|error| { + budget_failure(format!( + "failed to serialize Kubernetes object while calculating the TLS resource budget: {error}" + )) + })?; + to_u64(bytes.len()) + } + + fn validate_object_bytes(bytes: u64, label: &str) -> Result<(), TlsValidationFailure> { + let with_protocol_margin = checked_add(bytes, KUBERNETES_OBJECT_PROTOCOL_MARGIN_BYTES)?; + if with_protocol_margin > ETCD_DEFAULT_MAX_REQUEST_BYTES { + return Err(budget_failure(format!( + "{label} would serialize to at least {bytes} JSON bytes before the {KUBERNETES_OBJECT_PROTOCOL_MARGIN_BYTES}-byte Kubernetes/etcd protocol margin, exceeding the conservative {MAX_KUBERNETES_OBJECT_JSON_BYTES}-byte object budget" + ))); + } + Ok(()) + } + + fn custom_names_estimate( + entry: &TlsCertificateEntry, + ) -> Result { + let mut estimate = SanListEstimate::default(); + for name in entry + .hosts + .iter() + .chain(entry.cert_manager.dns_names.iter()) + .filter(|name| !name.is_empty()) + { + estimate.add_name(name)?; + } + Ok(estimate) + } + + fn status_hosts_estimate( + entry: &TlsCertificateEntry, + ) -> Result { + let mut estimate = SanListEstimate::default(); + for name in &entry.hosts { + estimate.add_name(name)?; + } + Ok(estimate) + } + + fn generated_names_estimate( + tenant: &Tenant, + namespace: &str, + cluster_domain: &str, + ) -> Result { + let tenant_name = tenant.name(); + let io_service = format!("{tenant_name}-io"); + let headless_service = tenant.headless_service_name(); + let mut estimate = SanListEstimate::default(); + for name in [ + format!("{io_service}.{namespace}.svc"), + cluster_dns::service_fqdn(&io_service, namespace, cluster_domain), + format!("{headless_service}.{namespace}.svc"), + cluster_dns::service_fqdn(&headless_service, namespace, cluster_domain), + ] { + estimate.add_name(&name)?; + } + add_pod_names(&mut estimate, tenant, namespace, cluster_domain, false)?; + Ok(estimate) + } + + fn required_names_estimate( + tenant: &Tenant, + namespace: &str, + cluster_domain: &str, + ) -> Result { + let headless_service = tenant.headless_service_name(); + let mut estimate = SanListEstimate::default(); + estimate.add_name(&cluster_dns::service_fqdn( + &headless_service, + namespace, + cluster_domain, + ))?; + add_pod_names(&mut estimate, tenant, namespace, cluster_domain, true)?; + Ok(estimate) + } + + fn add_pod_names( + estimate: &mut SanListEstimate, + tenant: &Tenant, + namespace: &str, + cluster_domain: &str, + skip_single_node_single_disk: bool, + ) -> Result<(), TlsValidationFailure> { + let tenant_name = tenant.name(); + let suffix = format!( + ".{}.{namespace}.svc.{cluster_domain}", + tenant.headless_service_name() + ); + for pool in &tenant.spec.pools { + if pool.servers <= 0 { + return Err(budget_failure(format!( + "spec.tls cannot expand DNS SANs for pool '{}' with servers={}; servers must be greater than 0", + pool.name, pool.servers + ))); + } + if skip_single_node_single_disk + && tenant.spec.pools.len() == 1 + && pool.is_single_node_single_disk() + { + continue; + } + + let count = pool.servers as u64; + let prefix = format!("{tenant_name}-{}-", pool.name); + let fixed_name_bytes = checked_add(to_u64(prefix.len())?, to_u64(suffix.len())?)?; + let total_name_bytes = checked_add( + checked_multiply(count, fixed_name_bytes)?, + ordinal_digit_sum(count)?, + )?; + estimate.add_ascii_batch(count, total_name_bytes)?; + } + Ok(()) + } + + fn ordinal_digit_sum(count: u64) -> Result { + let mut remaining = count; + let mut range = 10_u64; + let mut digits = 1_u64; + let mut total = 0_u64; + while remaining > 0 { + let in_range = remaining.min(range); + total = checked_add(total, checked_multiply(in_range, digits)?)?; + remaining -= in_range; + if remaining == 0 { + break; + } + // 0..=9 contains ten one-digit ordinals. Every subsequent decimal band contains + // 9 * 10^(digits - 1) ordinals (90 two-digit, 900 three-digit, ...). + range = if digits == 1 { + 90 + } else { + checked_multiply(range, 10)? + }; + digits = checked_add(digits, 1)?; + } + Ok(total) + } + + fn to_u64(value: usize) -> Result { + u64::try_from(value).map_err(|_| overflow_failure()) + } + + fn checked_add(left: u64, right: u64) -> Result { + left.checked_add(right).ok_or_else(overflow_failure) + } + + fn checked_multiply(left: u64, right: u64) -> Result { + left.checked_mul(right).ok_or_else(overflow_failure) + } + + fn overflow_failure() -> TlsValidationFailure { + budget_failure( + "spec.tls DNS SAN resource calculation overflowed before expansion".to_string(), + ) + } + + fn budget_failure(message: String) -> TlsValidationFailure { + TlsValidationFailure { + reason: Reason::CertificateInvalid, + message, + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn ordinal_digit_sum_matches_generated_ordinals() { + assert_eq!(ordinal_digit_sum(1), Ok(1)); + assert_eq!(ordinal_digit_sum(10), Ok(10)); + assert_eq!(ordinal_digit_sum(11), Ok(12)); + assert_eq!(ordinal_digit_sum(100), Ok(190)); + assert_eq!(ordinal_digit_sum(101), Ok(193)); + assert_eq!(ordinal_digit_sum(1000), Ok(2890)); + let expected_8192 = (0_u64..8192) + .map(|ordinal| ordinal.to_string().len() as u64) + .sum::(); + assert_eq!(ordinal_digit_sum(8192), Ok(expected_8192)); + } + + #[test] + fn ordinal_digit_sum_rejects_arithmetic_overflow() { + let failure = ordinal_digit_sum(u64::MAX) + .expect_err("unrepresentable aggregate name length must fail closed"); + assert!(failure.message.contains("overflowed before expansion")); + } + + #[test] + fn generated_name_estimate_conservatively_covers_serialized_output() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some("tenant-a".to_string()); + for servers in [101, 1000, 8192] { + tenant.spec.pools[0].servers = servers; + let budget = token(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN); + let estimate = generated_names_estimate( + &tenant, + "storage", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + ) + .expect("estimate should succeed"); + let generated = super::super::generated_dns_names(&budget); + let serialized = serde_json::to_vec(&generated).expect("SAN list should serialize"); + + assert_eq!(estimate.names, generated.len() as u64); + assert!( + estimate.array_bytes().expect("byte estimate") >= serialized.len() as u64, + "servers={servers} must use a conservative serialized-byte estimate" + ); + } + } + + #[test] + fn projected_tenant_budget_counts_existing_non_tls_object_bytes() { + let mut tenant = crate::tests::create_test_tenant(None, None); + let config = TlsConfig { + mode: TlsMode::CertManager, + require_san_match: false, + ..Default::default() + }; + tenant.spec.tls = Some(config.clone()); + let current_bytes = serde_json::to_vec(&tenant) + .expect("Tenant should serialize") + .len(); + let padding = MAX_KUBERNETES_OBJECT_JSON_BYTES as usize - current_bytes - 8 * 1024; + tenant.metadata.annotations = Some(std::collections::BTreeMap::from([( + "budget.test/padding".to_string(), + "x".repeat(padding), + )])); + let entries = vec![TlsCertificateEntry { + name: "default".to_string(), + default: true, + hosts: Vec::new(), + cert_manager: CertManagerTlsConfig::default(), + legacy: false, + }]; + + let failure = validate( + &tenant, + "storage", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + &config, + &entries, + ) + .expect_err("projected status must account for the existing complete Tenant object"); + + assert!(failure.message.contains("projected TLS status")); + assert!(failure.message.contains("protocol margin")); + } + + #[test] + fn complete_certificate_budget_counts_non_san_fields() { + let mut certificate = DynamicObject::new( + "tenant-a-server-tls", + &super::super::certificate_api_resource(), + ) + .data(json!({ + "spec": { + "secretName": "tenant-a-server-tls", + "commonName": "x".repeat(MAX_KUBERNETES_OBJECT_JSON_BYTES as usize), + "dnsNames": [] + } + })); + certificate.metadata.namespace = Some("storage".to_string()); + + let failure = validate_certificate_object(&certificate, "tenant-a-server-tls") + .expect_err("non-SAN Certificate fields must count toward the complete object"); + + assert!(failure.message.contains("cert-manager Certificate")); + assert!(failure.message.contains("object budget")); + } + + #[test] + fn certificate_entry_count_is_bounded_before_expansion() { + let tenant = crate::tests::create_test_tenant(None, None); + let config = TlsConfig { + mode: TlsMode::CertManager, + require_san_match: false, + ..Default::default() + }; + let entries = (0..=MAX_TLS_CERTIFICATE_ENTRIES) + .map(|index| TlsCertificateEntry { + name: format!("certificate-{index}"), + default: index == 0, + hosts: vec![format!("host-{index}.example.com")], + cert_manager: CertManagerTlsConfig { + include_generated_dns_names: Some(false), + ..Default::default() + }, + legacy: false, + }) + .collect::>(); + + let failure = validate( + &tenant, + "storage", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + &config, + &entries, + ) + .expect_err("certificate entry count must be bounded before SAN expansion"); + + assert!(failure.message.contains("certificate entries")); + } + + #[test] + fn status_budget_counts_hosts_and_all_default_dns_name_copies() { + let entry = TlsCertificateEntry { + name: "default".to_string(), + default: true, + hosts: vec!["api.example.com".to_string(), "雪.example.com".to_string()], + cert_manager: CertManagerTlsConfig { + dns_names: vec!["console.example.com".to_string()], + ..Default::default() + }, + legacy: false, + }; + let status = custom_names_estimate(&entry).expect("status estimate should succeed"); + let hosts = status_hosts_estimate(&entry).expect("host estimate should succeed"); + let mut estimated_bytes = 0_u64; + let mut estimated_names = 0_u64; + add_status_copy(&mut estimated_bytes, &mut estimated_names, status) + .expect("certificate dnsNames should fit"); + add_status_copy(&mut estimated_bytes, &mut estimated_names, hosts) + .expect("certificate hosts should fit"); + add_status_copy(&mut estimated_bytes, &mut estimated_names, status) + .expect("top-level default dnsNames should fit"); + + let dns_names = entry + .hosts + .iter() + .chain(entry.cert_manager.dns_names.iter()) + .cloned() + .collect::>(); + let actual_list_bytes = serde_json::to_vec(&dns_names) + .expect("dnsNames should serialize") + .len() as u64 + * 2 + + serde_json::to_vec(&entry.hosts) + .expect("hosts should serialize") + .len() as u64; + let actual_name_copies = dns_names.len() as u64 * 2 + entry.hosts.len() as u64; + + assert!(estimated_bytes >= actual_list_bytes); + assert!(estimated_names >= actual_name_copies); + } + + #[test] + fn byte_budgets_leave_explicit_etcd_object_margins() { + assert_eq!(ETCD_DEFAULT_MAX_REQUEST_BYTES, 1_572_864); + assert_eq!(KUBERNETES_OBJECT_PROTOCOL_MARGIN_BYTES, 65_536); + assert_eq!(MAX_CERTIFICATE_SAN_BYTES, 1_507_328); + assert_eq!(MAX_TLS_STATUS_SAN_BYTES, 524_288); + } + } +} + +use san_budget::ValidatedTlsSanBudget; + fn validate_managed_certificate_san_config( - tenant: &Tenant, - namespace: &str, + san_budget: &ValidatedTlsSanBudget<'_>, config: &TlsConfig, entry: &TlsCertificateEntry, - cluster_domain: &str, ) -> Result<(), TlsValidationFailure> { if !config.enable_internode_https || !entry.default || include_generated_dns_names(entry) { return Ok(()); } - let generated_names = required_tls_dns_names(tenant, namespace, cluster_domain); - let configured_names = certificate_dns_names( - tenant, - namespace, - &entry.cert_manager, - &entry.hosts, - false, - cluster_domain, - ) - .into_iter() - .collect::>(); - let missing_names = generated_names + let generated_names = required_tls_dns_names(san_budget); + let configured_names = + certificate_dns_names(san_budget, &entry.cert_manager, &entry.hosts, false) + .into_iter() + .collect::>(); + let configured_coverage = DnsSanCoverage::from_names( + configured_names.iter().map(String::as_str), + "configured TLS DNS names", + )?; + let missing_name = generated_names .iter() - .filter(|name| { - !configured_names - .iter() - .any(|configured| dns_name_covers(configured, name)) - }) - .cloned() - .collect::>(); + .find(|name| !configured_coverage.covers(name)); - if missing_names.is_empty() { + let Some(missing_name) = missing_name else { return Ok(()); - } + }; Err(TlsValidationFailure { reason: Reason::CertificateInvalid, message: format!( "spec.tls certificate '{}' cannot set certManager.includeGeneratedDnsNames=false while enableInternodeHttps=true unless certManager.dnsNames or hosts explicitly cover generated peer DNS names such as '{}'", - entry.name, missing_names[0] + entry.name, missing_name ), }) } fn san_validation_dns_names( - tenant: &Tenant, - namespace: &str, + san_budget: &ValidatedTlsSanBudget<'_>, config: &TlsConfig, entry: &TlsCertificateEntry, - cluster_domain: &str, ) -> Vec { let mut names = BTreeSet::new(); names.extend(entry.hosts.iter().filter(|name| !name.is_empty()).cloned()); @@ -1234,14 +2026,17 @@ fn san_validation_dns_names( .cloned(), ); if entry.default && config.enable_internode_https { - names.extend(required_tls_dns_names(tenant, namespace, cluster_domain)); + names.extend(required_tls_dns_names(san_budget)); } else if include_generated_dns_names(entry) { - names.extend(generated_dns_names(tenant, namespace, cluster_domain)); + names.extend(generated_dns_names(san_budget)); } names.into_iter().collect() } -fn required_tls_dns_names(tenant: &Tenant, namespace: &str, cluster_domain: &str) -> Vec { +fn required_tls_dns_names(san_budget: &ValidatedTlsSanBudget<'_>) -> Vec { + let tenant = san_budget.tenant(); + let namespace = san_budget.namespace(); + let cluster_domain = san_budget.cluster_domain(); let mut names = BTreeSet::new(); let tenant_name = tenant.name(); let headless_service = tenant.headless_service_name(); @@ -1267,7 +2062,10 @@ fn required_tls_dns_names(tenant: &Tenant, namespace: &str, cluster_domain: &str names.into_iter().collect() } -fn generated_dns_names(tenant: &Tenant, namespace: &str, cluster_domain: &str) -> Vec { +fn generated_dns_names(san_budget: &ValidatedTlsSanBudget<'_>) -> Vec { + let tenant = san_budget.tenant(); + let namespace = san_budget.namespace(); + let cluster_domain = san_budget.cluster_domain(); let mut names = BTreeSet::new(); let tenant_name = tenant.name(); let io_service = format!("{tenant_name}-io"); @@ -1298,20 +2096,56 @@ fn generated_dns_names(tenant: &Tenant, namespace: &str, cluster_domain: &str) - names.into_iter().collect() } -fn dns_name_covers(pattern: &str, dns_name: &str) -> bool { - if pattern == dns_name { - return true; +#[derive(Debug)] +struct DnsSanCoverage { + exact: BTreeSet, + wildcards: BTreeSet, +} + +impl DnsSanCoverage { + fn from_names<'a>( + names: impl IntoIterator, + label: &str, + ) -> Result { + let mut exact = BTreeSet::new(); + let mut wildcards = BTreeSet::new(); + let mut count = 0_u64; + for name in names { + count = count.checked_add(1).ok_or_else(|| TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!("{label} count overflowed the TLS SAN comparison budget"), + })?; + if count > MAX_TLS_SAN_NAMES_PER_LIST { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "{label} contains more than {MAX_TLS_SAN_NAMES_PER_LIST} names, exceeding the TLS SAN comparison allocation budget" + ), + }); + } + let normalized = name.to_ascii_lowercase(); + if is_dns_wildcard_pattern(&normalized) { + wildcards.insert(normalized); + } else { + exact.insert(normalized); + } + } + Ok(Self { exact, wildcards }) + } + + fn covers(&self, dns_name: &str) -> bool { + let normalized = dns_name.to_ascii_lowercase(); + if is_dns_wildcard_pattern(&normalized) { + return self.wildcards.contains(&normalized); + } + if self.exact.contains(&normalized) { + return true; + } + let Some((_, suffix)) = normalized.split_once('.') else { + return false; + }; + self.wildcards.contains(&format!("*.{suffix}")) } - let Some(suffix) = pattern.strip_prefix("*.") else { - return false; - }; - let Some(label) = dns_name - .strip_suffix(suffix) - .and_then(|prefix| prefix.strip_suffix('.')) - else { - return false; - }; - !label.is_empty() && !label.contains('.') } fn is_dns_wildcard_pattern(name: &str) -> bool { @@ -1491,84 +2325,158 @@ fn supported_tls_secret_type(secret_type: &str) -> bool { ) } -fn validate_tls_secret_san_match( +fn validate_tls_secret_san_match_with_budget( secret_name: &str, cert_bytes: &[u8], expected_dns_names: &[String], + runtime_budget: &mut TlsCertificateRuntimeBudget, ) -> Result<(), TlsValidationFailure> { if expected_dns_names.is_empty() { return Ok(()); } + if expected_dns_names.len() as u64 > MAX_TLS_SAN_COVERAGE_LOOKUPS { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "TLS SAN validation requires {} DNS name lookups, exceeding the comparison budget of {MAX_TLS_SAN_COVERAGE_LOOKUPS}", + expected_dns_names.len() + ), + }); + } - let certs = rustls_pemfile::certs(&mut Cursor::new(cert_bytes)) - .collect::>, _>>() + let mut cursor = Cursor::new(cert_bytes); + let mut certs = rustls_pemfile::certs(&mut cursor); + let cert_der = certs + .next() + .transpose() .map_err(|_| TlsValidationFailure { reason: Reason::CertificateInvalid, message: format!( "TLS certificate in Secret '{}' key '{}' must contain a valid PEM certificate", secret_name, TLS_CERT_KEY ), - })?; - let cert_der = certs.first().ok_or_else(|| TlsValidationFailure { + })? + .ok_or_else(|| TlsValidationFailure { reason: Reason::CertificateInvalid, message: format!( "TLS certificate in Secret '{}' key '{}' must contain at least one valid PEM certificate", secret_name, TLS_CERT_KEY ), })?; - let cert = webpki::EndEntityCert::try_from(cert_der).map_err(|_| TlsValidationFailure { + let leaf_der_bytes = runtime_budget.charge_der(cert_der.as_ref().len())?; + // Validate the remaining PEM certificate blocks without retaining the chain in memory. + for certificate in certs { + let certificate = certificate.map_err(|_| TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "TLS certificate in Secret '{}' key '{}' must contain a valid PEM certificate", + secret_name, TLS_CERT_KEY + ), + })?; + runtime_budget.charge_der(certificate.as_ref().len())?; + } + let cert = webpki::EndEntityCert::try_from(&cert_der).map_err(|_| TlsValidationFailure { reason: Reason::CertificateInvalid, message: format!( "TLS certificate in Secret '{}' key '{}' must be a valid X.509 end-entity certificate", secret_name, TLS_CERT_KEY ), })?; - let presented_dns_names = cert.valid_dns_names().collect::>(); + let concrete_expected_count = expected_dns_names + .iter() + .filter(|name| !is_dns_wildcard_pattern(name)) + .count() as u64; + runtime_budget.charge_scan(leaf_der_bytes, concrete_expected_count)?; + + let mut presented_count = 0_u64; + let mut presented_wildcards = BTreeSet::new(); + for presented in cert.valid_dns_names() { + presented_count = presented_count.checked_add(1).ok_or_else(|| { + TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "TLS certificate in Secret '{secret_name}' SAN count overflowed the comparison budget" + ), + } + })?; + if presented_count > MAX_TLS_SAN_NAMES_PER_LIST { + return Err(TlsValidationFailure { + reason: Reason::CertificateInvalid, + message: format!( + "TLS certificate in Secret '{secret_name}' contains more than {MAX_TLS_SAN_NAMES_PER_LIST} DNS names, exceeding the TLS SAN comparison allocation budget" + ), + }); + } + if is_dns_wildcard_pattern(presented) { + presented_wildcards.insert(presented.to_ascii_lowercase()); + } + } - let mut missing = Vec::new(); + let mut missing_count = 0_u64; + let mut missing_examples = Vec::new(); for dns_name in expected_dns_names { - if is_dns_wildcard_pattern(dns_name) { - if !presented_dns_names - .iter() - .any(|presented| presented.eq_ignore_ascii_case(dns_name)) - { - missing.push(dns_name.clone()); + let covered = if is_dns_wildcard_pattern(dns_name) { + presented_wildcards.contains(&dns_name.to_ascii_lowercase()) + } else { + let server_name = + ServerName::try_from(dns_name.as_str()).map_err(|_| TlsValidationFailure { + reason: Reason::CertificateSanMismatch, + message: format!("required TLS DNS name '{dns_name}' is invalid"), + })?; + cert.verify_is_valid_for_subject_name(&server_name).is_ok() + }; + if !covered { + missing_count += 1; + if missing_examples.len() < 3 { + missing_examples.push(dns_name.as_str()); } - continue; - } - let server_name = - ServerName::try_from(dns_name.as_str()).map_err(|_| TlsValidationFailure { - reason: Reason::CertificateSanMismatch, - message: format!("required TLS DNS name '{dns_name}' is invalid"), - })?; - if cert.verify_is_valid_for_subject_name(&server_name).is_err() { - missing.push(dns_name.clone()); } } - if missing.is_empty() { + if missing_count == 0 { Ok(()) } else { + let omitted = missing_count.saturating_sub(missing_examples.len() as u64); + let detail = if omitted == 0 { + missing_examples.join(", ") + } else { + format!("{} (and {omitted} more)", missing_examples.join(", ")) + }; Err(TlsValidationFailure { reason: Reason::CertificateSanMismatch, message: format!( "TLS certificate in Secret '{}' key '{}' does not cover required DNS names: {}", - secret_name, - TLS_CERT_KEY, - missing.join(", ") + secret_name, TLS_CERT_KEY, detail ), }) } } -fn certificate_secret_ca_material( - secret: &Secret, +#[cfg(test)] +fn validate_tls_secret_san_match( + secret_name: &str, + cert_bytes: &[u8], + expected_dns_names: &[String], +) -> Result<(), TlsValidationFailure> { + let mut runtime_budget = TlsCertificateRuntimeBudget::default(); + runtime_budget.charge_secret_material(cert_bytes.len())?; + validate_tls_secret_san_match_with_budget( + secret_name, + cert_bytes, + expected_dns_names, + &mut runtime_budget, + ) +} + +fn certificate_ca_material( + ca_bytes: Option<&[u8]>, secret_name: &str, enable_internode_https: bool, trust_system_ca: bool, + runtime_budget: &mut TlsCertificateRuntimeBudget, ) -> Result, TlsValidationFailure> { - if let Some(ca_bytes) = secret_bytes(secret, CA_CERT_KEY) { - validate_ca_bundle_bytes(secret_name, CA_CERT_KEY, ca_bytes)?; + if let Some(ca_bytes) = ca_bytes { + validate_ca_bundle_bytes(secret_name, CA_CERT_KEY, ca_bytes, runtime_budget)?; return Ok(Some(ServerCaMaterial { key: CA_CERT_KEY.to_string(), bytes: ca_bytes.to_vec(), @@ -1588,21 +2496,50 @@ fn certificate_secret_ca_material( Ok(None) } +#[cfg(test)] +fn certificate_secret_ca_material( + secret: &Secret, + secret_name: &str, + enable_internode_https: bool, + trust_system_ca: bool, +) -> Result, TlsValidationFailure> { + let mut runtime_budget = TlsCertificateRuntimeBudget::default(); + certificate_ca_material( + secret_bytes(secret, CA_CERT_KEY), + secret_name, + enable_internode_https, + trust_system_ca, + &mut runtime_budget, + ) +} + fn validate_ca_bundle_bytes( secret_name: &str, key: &str, bytes: &[u8], + runtime_budget: &mut TlsCertificateRuntimeBudget, ) -> Result<(), TlsValidationFailure> { - let parsed = rustls_pemfile::certs(&mut Cursor::new(bytes)).collect::, _>>(); - match parsed { - Ok(certs) if !certs.is_empty() => Ok(()), - Ok(_) | Err(_) => Err(TlsValidationFailure { - reason: Reason::CaBundleInvalid, - message: format!( - "CA bundle in Secret '{}' key '{}' must contain at least one valid PEM certificate", - secret_name, key - ), - }), + let mut cursor = Cursor::new(bytes); + let mut parsed = rustls_pemfile::certs(&mut cursor); + let Some(first) = parsed.next() else { + return Err(invalid_ca_bundle(secret_name, key)); + }; + let first = first.map_err(|_| invalid_ca_bundle(secret_name, key))?; + runtime_budget.charge_der(first.as_ref().len())?; + for certificate in parsed { + let certificate = certificate.map_err(|_| invalid_ca_bundle(secret_name, key))?; + runtime_budget.charge_der(certificate.as_ref().len())?; + } + Ok(()) +} + +fn invalid_ca_bundle(secret_name: &str, key: &str) -> TlsValidationFailure { + TlsValidationFailure { + reason: Reason::CaBundleInvalid, + message: format!( + "CA bundle in Secret '{}' key '{}' must contain at least one valid PEM certificate", + secret_name, key + ), } } @@ -1726,7 +2663,7 @@ fn cert_manager_tls_status( let server_secret_ref = SecretStatusRef { name: default_certificate.secret_name.clone(), key: None, - resource_version: default_certificate.secret.metadata.resource_version.clone(), + resource_version: default_certificate.secret_resource_version.clone(), }; TlsCertificateStatus { mode: tls_mode_name(config.mode).to_string(), @@ -1740,11 +2677,7 @@ fn cert_manager_tls_status( .iter() .map(tls_server_certificate_status) .collect(), - ca_secret_ref: ca_status_ref( - &default_certificate.secret_name, - &default_certificate.secret, - explicit_ca, - ), + ca_secret_ref: ca_status_ref(default_certificate, explicit_ca), client_ca_secret_ref: client_ca.map(|(secret_ref, ca_secret)| SecretStatusRef { name: secret_ref.name.clone(), key: Some(secret_ref.key.clone()), @@ -1772,15 +2705,14 @@ fn tls_server_certificate_status( server_secret_ref: SecretStatusRef { name: certificate.secret_name.clone(), key: None, - resource_version: certificate.secret.metadata.resource_version.clone(), + resource_version: certificate.secret_resource_version.clone(), }, dns_names: certificate.san_dns_names.clone(), } } fn ca_status_ref( - secret_name: &str, - secret: &Secret, + default_certificate: &ObservedTlsCertificate, explicit_ca: Option<(&SecretKeyReference, &Secret)>, ) -> Option { if let Some((secret_ref, ca_secret)) = explicit_ca { @@ -1790,11 +2722,14 @@ fn ca_status_ref( resource_version: ca_secret.metadata.resource_version.clone(), }); } - secret_bytes(secret, CA_CERT_KEY).map(|_| SecretStatusRef { - name: secret_name.to_string(), - key: Some(CA_CERT_KEY.to_string()), - resource_version: secret.metadata.resource_version.clone(), - }) + default_certificate + .ca_bytes + .as_ref() + .map(|_| SecretStatusRef { + name: default_certificate.secret_name.clone(), + key: Some(CA_CERT_KEY.to_string()), + resource_version: default_certificate.secret_resource_version.clone(), + }) } #[cfg(test)] @@ -1873,23 +2808,14 @@ fn tls_hash( hash_str( &mut hasher, "serverSecret.resourceVersion", - certificate - .secret - .metadata - .resource_version - .as_deref() - .unwrap_or(""), - ); - hash_bytes( - &mut hasher, - "tls.crt", - secret_bytes(&certificate.secret, TLS_CERT_KEY), + certificate.secret_resource_version.as_deref().unwrap_or(""), ); + hash_bytes(&mut hasher, "tls.crt", Some(&certificate.cert_bytes)); if certificate.entry.default { hash_bytes( &mut hasher, "secret.ca.crt", - secret_bytes(&certificate.secret, CA_CERT_KEY), + certificate.ca_bytes.as_deref(), ); } } @@ -2039,6 +2965,156 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj ); } + #[test] + fn tls_san_budget_allows_external_secret_without_expansion() { + let tenant = tenant_with_pool_servers(&[i32::MAX]); + let config = TlsConfig { + mode: TlsMode::CertManager, + require_san_match: false, + ..Default::default() + }; + let entries = vec![tls_budget_entry(false, Some(false))]; + + let budget = san_budget::validate( + &tenant, + "storage", + cluster_dns::DEFAULT_CLUSTER_DOMAIN, + &config, + &entries, + ) + .expect("zero-expansion external Secret should not inherit a topology limit"); + + assert_eq!(budget.tenant().name(), tenant.name()); + } + + #[test] + fn tls_san_budget_rejects_oversized_status_before_expansion() { + let mut tenant = tenant_with_pool_servers(&[800]); + tenant.metadata.name = Some("tenant-long".to_string()); + let config = TlsConfig { + mode: TlsMode::CertManager, + require_san_match: false, + ..Default::default() + }; + let entries = vec![tls_budget_entry(false, None)]; + let namespace = "n".repeat(63); + let cluster_domain = format!("{}a", "a.".repeat(119)); + + let failure = san_budget::validate(&tenant, &namespace, &cluster_domain, &config, &entries) + .expect_err("duplicated default status SAN bytes must be bounded before expansion"); + + assert!(failure.message.contains("Tenant status")); + assert!(failure.message.contains("derived from etcd")); + } + + #[test] + fn tls_san_budget_rejects_oversized_managed_certificate_status_before_expansion() { + let mut tenant = tenant_with_pool_servers(&[4000]); + tenant.metadata.name = Some("tenant-long".to_string()); + let config = TlsConfig { + mode: TlsMode::CertManager, + require_san_match: false, + ..Default::default() + }; + let entries = vec![tls_budget_entry(true, None)]; + let namespace = "n".repeat(63); + let cluster_domain = format!("{}a", "a.".repeat(119)); + + let failure = san_budget::validate(&tenant, &namespace, &cluster_domain, &config, &entries) + .expect_err("oversized managed certificate status must fail before expansion"); + + assert!(failure.message.contains("TLS status")); + assert!(failure.message.contains("derived from etcd")); + } + + #[test] + fn reconcile_wide_certificate_runtime_budget_accumulates_across_certificates() { + let mut budget = TlsCertificateRuntimeBudget::default(); + let der_bytes = 1024_usize; + let concrete_names = MAX_TLS_SAN_VERIFICATION_SCAN_BYTES / 2 / der_bytes as u64; + + let first_der = budget + .charge_der(der_bytes) + .expect("one certificate should fit the reconcile-wide DER budget"); + budget + .charge_scan(first_der, concrete_names) + .expect("one certificate should fit the reconcile-wide scan budget"); + let second_der = budget + .charge_der(der_bytes) + .expect("a second small certificate should fit the DER budget"); + let failure = budget + .charge_scan(second_der, concrete_names) + .expect_err("the second certificate must consume the same shared scan budget"); + + assert!( + failure + .message + .contains("reconcile-wide verification budget") + ); + } + + #[test] + fn reconcile_wide_secret_material_budget_includes_private_keys() { + let mut budget = TlsCertificateRuntimeBudget::default(); + let certificate_bytes = MAX_TLS_SECRET_MATERIAL_BYTES as usize / 2; + let private_key_bytes = MAX_TLS_SECRET_MATERIAL_BYTES as usize - certificate_bytes; + + budget + .charge_secret_material(certificate_bytes) + .expect("certificate PEM should fit half the shared budget"); + budget + .charge_secret_material(private_key_bytes) + .expect("private-key PEM should consume the rest of the shared budget"); + let failure = budget + .charge_secret_material(1) + .expect_err("secret material from another certificate must exceed the shared budget"); + + assert!(failure.message.contains("reconcile-wide budget")); + assert!(failure.message.contains("secret-material")); + } + + #[test] + fn reconcile_wide_der_budget_is_cumulative_across_parsed_bundles() { + let mut budget = TlsCertificateRuntimeBudget::default(); + + budget + .charge_der(MAX_TLS_CERTIFICATE_DER_BYTES as usize) + .expect("the exact DER budget should be accepted"); + let failure = budget + .charge_der(1) + .expect_err("additional parsed DER bytes must exceed the shared budget"); + + assert!(failure.message.contains("reconcile-wide budget")); + } + + #[test] + fn dns_san_coverage_indexes_exact_and_single_label_wildcards() { + let coverage = + DnsSanCoverage::from_names(["API.EXAMPLE.COM", "*.storage.example.com"], "test SANs") + .expect("small SAN set should index"); + + assert!(coverage.covers("api.example.com")); + assert!(coverage.covers("pod.storage.example.com")); + assert!(coverage.covers("*.storage.example.com")); + assert!(!coverage.covers("nested.pod.storage.example.com")); + assert!(!coverage.covers("storage.example.com")); + } + + #[test] + fn dns_san_coverage_rejects_oversized_configured_set() { + let names = (0..=MAX_TLS_SAN_NAMES_PER_LIST) + .map(|index| format!("name-{index}.example.com")) + .collect::>(); + + let failure = DnsSanCoverage::from_names( + names.iter().map(String::as_str), + "presented certificate SANs", + ) + .expect_err("presented SAN index allocation must be bounded"); + + assert!(failure.message.contains("comparison allocation budget")); + } + #[test] fn default_secret_type_rejects_unconventional_server_secret() { let secret = tls_secret("server-tls", "7", Some("Opaque"), true, true, None); @@ -2289,6 +3365,41 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj ); } + #[test] + fn require_san_match_preserves_webpki_absolute_dns_name_semantics() { + assert_eq!( + validate_tls_secret_san_match( + "server-tls", + CERT_WITH_PEER_SANS_PEM, + &["localhost.".to_string()], + ), + Ok(()) + ); + } + + #[test] + fn require_san_match_bounds_scans_across_non_dns_sans() { + let ip_sans = (0..1024) + .map(|index| format!("10.{}.{}.1", index / 256, index % 256)) + .collect::>(); + let certificate = rcgen::generate_simple_self_signed(ip_sans) + .expect("IP subjectAltNames should generate a certificate"); + let expected_dns_names = (0..4096) + .map(|index| format!("peer-{index}.example.com")) + .collect::>(); + + let failure = validate_tls_secret_san_match( + "server-tls", + certificate.cert.pem().as_bytes(), + &expected_dns_names, + ) + .expect_err("non-DNS GeneralNames must be included in the scan-work budget"); + + assert_eq!(failure.reason, Reason::CertificateInvalid); + assert!(failure.message.contains("certificate DER bytes")); + assert!(failure.message.contains("verification budget")); + } + #[test] fn require_san_match_rejects_certificate_missing_required_peer_dns_names() { let expected_dns_names = vec![ @@ -2426,14 +3537,14 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj let Some(cert_manager) = config.cert_manager.as_ref() else { panic!("test config must include cert-manager settings"); }; + let san_budget = + validated_test_san_budget(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN); let certificate = build_cert_manager_certificate( - &tenant, - "storage", + &san_budget, cert_manager, &[], true, - cluster_dns::DEFAULT_CLUSTER_DOMAIN, CertManagerCertificateNames { secret: "tenant-a-server-tls", certificate: "tenant-a-server", @@ -2518,14 +3629,13 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj tenant.metadata.namespace = Some("mse".to_string()); tenant.spec.pools[0].name = "mse-nvme-500".to_string(); tenant.spec.pools[0].servers = 3; + let san_budget = validated_test_san_budget(&tenant, "mse", "k8s.mse.cloud"); let certificate = build_cert_manager_certificate( - &tenant, - "mse", + &san_budget, &CertManagerTlsConfig::default(), &[], true, - "k8s.mse.cloud", CertManagerCertificateNames { secret: "prod-rustfs-private-certificate-secret", certificate: "prod-rustfs-private-certificate", @@ -2567,14 +3677,14 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj include_generated_dns_names: Some(false), ..Default::default() }; + let san_budget = + validated_test_san_budget(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN); let certificate = build_cert_manager_certificate( - &tenant, - "storage", + &san_budget, &cert_manager, &["s3.example.com".to_string()], false, - cluster_dns::DEFAULT_CLUSTER_DOMAIN, CertManagerCertificateNames { secret: "tenant-a-public-tls", certificate: "tenant-a-public-tls", @@ -2673,14 +3783,14 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj cert_manager: CertManagerTlsConfig::default(), legacy: false, }; + let san_budget = + validated_test_san_budget(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN); let certificate = build_cert_manager_certificate( - &tenant, - "storage", + &san_budget, &entry.cert_manager, &entry.hosts, include_generated_dns_names(&entry), - cluster_dns::DEFAULT_CLUSTER_DOMAIN, CertManagerCertificateNames { secret: "tenant-a-public-tls", certificate: "tenant-a-public-tls", @@ -2717,14 +3827,14 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj }, legacy: false, }; + let san_budget = + validated_test_san_budget(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN); let certificate = build_cert_manager_certificate( - &tenant, - "storage", + &san_budget, &entry.cert_manager, &entry.hosts, include_generated_dns_names(&entry), - cluster_dns::DEFAULT_CLUSTER_DOMAIN, CertManagerCertificateNames { secret: "tenant-a-public-tls", certificate: "tenant-a-public-tls", @@ -2758,11 +3868,12 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj enable_internode_https: true, ..Default::default() }; - let generated_pod_dns = - generated_dns_names(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN) - .into_iter() - .find(|name| name.contains("-0.")) - .expect("generated names should include a pod DNS name"); + let san_budget = + validated_test_san_budget(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN); + let generated_pod_dns = generated_dns_names(&san_budget) + .into_iter() + .find(|name| name.contains("-0.")) + .expect("generated names should include a pod DNS name"); let default_entry = TlsCertificateEntry { name: "internal".to_string(), @@ -2774,13 +3885,7 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj }, legacy: false, }; - let names = san_validation_dns_names( - &tenant, - "storage", - &config, - &default_entry, - cluster_dns::DEFAULT_CLUSTER_DOMAIN, - ); + let names = san_validation_dns_names(&san_budget, &config, &default_entry); assert!(names.contains(&generated_pod_dns)); let non_default_entry = TlsCertificateEntry { @@ -2793,13 +3898,7 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj }, legacy: false, }; - let names = san_validation_dns_names( - &tenant, - "storage", - &config, - &non_default_entry, - cluster_dns::DEFAULT_CLUSTER_DOMAIN, - ); + let names = san_validation_dns_names(&san_budget, &config, &non_default_entry); assert!(names.contains(&"s3.example.com".to_string())); assert!(names.contains(&generated_pod_dns)); @@ -2807,13 +3906,7 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj cert_manager: CertManagerTlsConfig::default(), ..non_default_entry }; - let names = san_validation_dns_names( - &tenant, - "storage", - &config, - &public_default, - cluster_dns::DEFAULT_CLUSTER_DOMAIN, - ); + let names = san_validation_dns_names(&san_budget, &config, &public_default); assert_eq!(names, vec!["s3.example.com".to_string()]); } @@ -2838,15 +3931,11 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj }, legacy: false, }; + let san_budget = + validated_test_san_budget(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN); - let failure = validate_managed_certificate_san_config( - &tenant, - "storage", - &config, - &entry, - cluster_dns::DEFAULT_CLUSTER_DOMAIN, - ) - .expect_err("managed internode cert must cover generated peer DNS names"); + let failure = validate_managed_certificate_san_config(&san_budget, &config, &entry) + .expect_err("managed internode cert must cover generated peer DNS names"); assert_eq!(failure.reason, Reason::CertificateInvalid); assert!( @@ -2857,16 +3946,9 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj failure.message ); - entry.cert_manager.dns_names = - generated_dns_names(&tenant, "storage", cluster_dns::DEFAULT_CLUSTER_DOMAIN); + entry.cert_manager.dns_names = generated_dns_names(&san_budget); assert_eq!( - validate_managed_certificate_san_config( - &tenant, - "storage", - &config, - &entry, - cluster_dns::DEFAULT_CLUSTER_DOMAIN, - ), + validate_managed_certificate_san_config(&san_budget, &config, &entry), Ok(()) ); } @@ -2897,20 +3979,14 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj }, legacy: false, }; + let san_budget = validated_test_san_budget(&tenant, "mse", "k8s.mse.cloud"); assert_eq!( - validate_managed_certificate_san_config( - &tenant, - "mse", - &config, - &entry, - "k8s.mse.cloud", - ), + validate_managed_certificate_san_config(&san_budget, &config, &entry), Ok(()) ); - let expected_dns_names = - san_validation_dns_names(&tenant, "mse", &config, &entry, "k8s.mse.cloud"); + let expected_dns_names = san_validation_dns_names(&san_budget, &config, &entry); assert!( expected_dns_names .iter() @@ -3292,6 +4368,54 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj } } + fn tenant_with_pool_servers(servers: &[i32]) -> Tenant { + let mut tenant = crate::tests::create_test_tenant(None, None); + let template = tenant.spec.pools[0].clone(); + tenant.spec.pools = servers + .iter() + .enumerate() + .map(|(index, servers)| { + let mut pool = template.clone(); + pool.name = format!("pool-{index}"); + pool.servers = *servers; + pool + }) + .collect(); + tenant + } + + fn tls_budget_entry( + manage_certificate: bool, + include_generated_dns_names: Option, + ) -> TlsCertificateEntry { + TlsCertificateEntry { + name: "default".to_string(), + default: true, + hosts: Vec::new(), + cert_manager: CertManagerTlsConfig { + manage_certificate, + include_generated_dns_names, + ..Default::default() + }, + legacy: false, + } + } + + fn validated_test_san_budget<'a>( + tenant: &'a Tenant, + namespace: &'a str, + cluster_domain: &'a str, + ) -> ValidatedTlsSanBudget<'a> { + let config = TlsConfig { + mode: TlsMode::CertManager, + require_san_match: false, + ..Default::default() + }; + let entries = vec![tls_budget_entry(false, None)]; + san_budget::validate(tenant, namespace, cluster_domain, &config, &entries) + .expect("test SAN expansion should fit the production budget") + } + fn observed_tls_certificate( name: &str, default: bool, @@ -3301,6 +4425,11 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj cert_manager: CertManagerTlsConfig, san_dns_names: Vec<&str>, ) -> ObservedTlsCertificate { + let secret_resource_version = secret.metadata.resource_version.clone(); + let cert_bytes = secret_bytes(&secret, TLS_CERT_KEY) + .map(ToOwned::to_owned) + .unwrap_or_default(); + let ca_bytes = secret_bytes(&secret, CA_CERT_KEY).map(ToOwned::to_owned); ObservedTlsCertificate { entry: TlsCertificateEntry { name: name.to_string(), @@ -3310,7 +4439,9 @@ S2+cuFyHX+xgTPNxiG9zUDrgtXds/63ePISjIADAUvsmI97k96E6jdcgB9MmWdJj legacy: false, }, secret_name: secret_name.to_string(), - secret, + secret_resource_version, + cert_bytes, + ca_bytes, certificate_ref: None, san_dns_names: san_dns_names.into_iter().map(ToString::to_string).collect(), }