Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions charts/operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,11 @@ defaults:
# always-on enforce-redirect egress guard (proxy-sidecar / lite). MUST match
# the authbridge listener.transparent_proxy_addr (default :8082).
transparentPort: 8082
# INBOUND transparent listener port — the PREROUTING REDIRECT target when a
# workload selects inboundInterception: transparent. MUST match the authbridge
# listener.transparent_inbound_addr (preset default :8083), and must differ
# from transparentPort (they are separate listeners in the same container).
transparentInboundPort: 8083
# Cluster DNS is kept direct by proxy-init itself (it reads the pod's
# /etc/resolv.conf nameservers), so there is no in-cluster CIDR knob to set —
# works on Kind / OpenShift / EKS / NodeLocal-DNSCache with no per-cluster config.
Expand All @@ -280,6 +285,16 @@ defaults:
allowedEgressEnforcement:
- enforce-redirect
- none
# Which inbound interception mechanisms workloads in this cluster may select.
# A resolved value outside this list falls back to the FIRST entry, so order
# matters. Transparent inbound needs a privileged proxy-init container
# (NET_ADMIN), so an admin may want to forbid or mandate it:
# ["reverse-proxy"] — port stealing only, no NET_ADMIN
# ["transparent"] — hard inbound boundary required
# ["reverse-proxy", "transparent"] — workloads choose (default)
allowedInboundInterception:
- reverse-proxy
- transparent

# Resource defaults (conservative for dev)
# Note: requests must be <= limits
Expand Down
9 changes: 8 additions & 1 deletion operator/internal/webhook/config/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,20 @@ func CompiledDefaults() *PlatformConfig {
// Transparent listener port — must match the authbridge proxy-sidecar
// preset (listener.transparent_proxy_addr default :8082).
TransparentPort: 8082,
// Inbound transparent listener. Matches the authbridge proxy-sidecar
// preset (listener.transparent_inbound_addr default :8083); 8080/8081/8082
// are the reverse, forward and transparent-egress listeners.
TransparentInboundPort: 8083,
// Empty by default: proxy-init auto-detects the iptables backend from
// /proc/modules. Set (e.g. "iptables") to force a backend per-platform.
IptablesCmd: "",
// Both modes allowed by default. Set to ["none"] on platforms
// where iptables is unavailable (ROSA HCP, managed OpenShift),
// or ["enforce-redirect"] to prevent opt-out.
AllowedEgressEnforcement: []string{"enforce-redirect", "none"},
// Both allowed by default: transparent inbound is opt-in per workload,
// and a platform admin can narrow this to forbid or mandate it.
AllowedInboundInterception: []string{"reverse-proxy", "transparent"},
AllowedEgressEnforcement: []string{"enforce-redirect", "none"},
},
Resources: ResourcesConfig{
EnvoyProxy: corev1.ResourceRequirements{
Expand Down
107 changes: 107 additions & 0 deletions operator/internal/webhook/config/transparent_inbound_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package config

import "testing"

// TestValidate_TransparentInboundPort covers the misconfigurations that would
// otherwise surface as a pod that passes admission and then fails to start.
func TestValidate_TransparentInboundPort(t *testing.T) {
tests := []struct {
name string
mutate func(*PlatformConfig)
wantErr bool
}{
{
name: "defaults are valid",
mutate: func(*PlatformConfig) {},
},
{
name: "below the privileged range",
mutate: func(c *PlatformConfig) { c.Proxy.TransparentInboundPort = 80 },
wantErr: true,
},
{
name: "above the port range",
mutate: func(c *PlatformConfig) { c.Proxy.TransparentInboundPort = 70000 },
wantErr: true,
},
{
name: "unset (zero) is rejected rather than silently defaulted",
mutate: func(c *PlatformConfig) { c.Proxy.TransparentInboundPort = 0 },
wantErr: true,
},
{
// Both listeners live in one container, so a shared value makes the
// second bind fail at pod start — long after admission succeeded.
name: "colliding with the egress transparent port",
mutate: func(c *PlatformConfig) {
c.Proxy.TransparentInboundPort = c.Proxy.TransparentPort
},
wantErr: true,
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cfg := CompiledDefaults()
tc.mutate(cfg)
err := cfg.Validate()
if tc.wantErr && err == nil {
t.Fatal("expected a validation error, got nil")
}
if !tc.wantErr && err != nil {
t.Fatalf("unexpected validation error: %v", err)
}
})
}
}

func TestValidate_AllowedInboundInterception(t *testing.T) {
tests := []struct {
name string
allowed []string
wantErr bool
}{
{name: "both", allowed: []string{"reverse-proxy", "transparent"}},
{name: "forbid transparent", allowed: []string{"reverse-proxy"}},
{name: "mandate transparent", allowed: []string{"transparent"}},
{
// An empty list would make the fallback index panic, and "allow
// nothing" has no sensible meaning.
name: "empty is rejected", allowed: []string{}, wantErr: true,
},
{name: "unknown value is rejected", allowed: []string{"reverse-proxy", "tranparent"}, wantErr: true},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cfg := CompiledDefaults()
cfg.Proxy.AllowedInboundInterception = tc.allowed
err := cfg.Validate()
if tc.wantErr && err == nil {
t.Fatal("expected a validation error, got nil")
}
if !tc.wantErr && err != nil {
t.Fatalf("unexpected validation error: %v", err)
}
})
}
}

// TestCompiledDefaults_TransparentInboundDefaults pins the values that must stay
// in lockstep with authbridge's proxy-sidecar preset; a drift here silently
// redirects inbound traffic to a dead port.
func TestCompiledDefaults_TransparentInboundDefaults(t *testing.T) {
cfg := CompiledDefaults()
if cfg.Proxy.TransparentInboundPort != 8083 {
t.Errorf("TransparentInboundPort = %d, want 8083 (authbridge preset transparent_inbound_addr)", cfg.Proxy.TransparentInboundPort)
}
if len(cfg.Proxy.AllowedInboundInterception) != 2 {
t.Errorf("AllowedInboundInterception = %v, want both mechanisms allowed by default", cfg.Proxy.AllowedInboundInterception)
}
// Order matters: the first entry is the fallback when a workload requests a
// value outside the list, and the no-privilege shape must win.
if cfg.Proxy.AllowedInboundInterception[0] != "reverse-proxy" {
t.Errorf("AllowedInboundInterception[0] = %q, want reverse-proxy (the fallback must not grant NET_ADMIN)",
cfg.Proxy.AllowedInboundInterception[0])
}
}
35 changes: 35 additions & 0 deletions operator/internal/webhook/config/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,20 @@ type ProxyConfig struct {
// where auto-detection is wrong or undesired.
IptablesCmd string `json:"iptablesCmd" yaml:"iptablesCmd"`

// TransparentInboundPort is the INBOUND transparent listener port — the
// PREROUTING REDIRECT target when inboundInterception is "transparent".
// MUST match the authbridge listener.transparent_inbound_addr (preset
// default :8083); a mismatch redirects inbound traffic to a dead port.
TransparentInboundPort int32 `json:"transparentInboundPort" yaml:"transparentInboundPort"`

// AllowedInboundInterception restricts which inboundInterception values
// workloads in this cluster may select, mirroring
// AllowedEgressEnforcement. Transparent inbound requires a privileged
// proxy-init container, so a platform admin may want to forbid it
// (["reverse-proxy"]) or mandate it (["transparent"]). A resolved value
// outside the list falls back to the list's first entry.
AllowedInboundInterception []string `json:"allowedInboundInterception,omitempty" yaml:"allowedInboundInterception,omitempty"`

// AllowedEgressEnforcement restricts which egressEnforcement values
// workloads (AgentRuntime CR / namespace ConfigMap) may select.
// The webhook rejects resolved values not in this list, falling back
Expand Down Expand Up @@ -108,6 +122,10 @@ func (c *PlatformConfig) DeepCopy() *PlatformConfig {
copy(result.TokenExchange.DefaultScopes, c.TokenExchange.DefaultScopes)
}

if c.Proxy.AllowedInboundInterception != nil {
result.Proxy.AllowedInboundInterception = make([]string, len(c.Proxy.AllowedInboundInterception))
copy(result.Proxy.AllowedInboundInterception, c.Proxy.AllowedInboundInterception)
}
if c.Proxy.AllowedEgressEnforcement != nil {
result.Proxy.AllowedEgressEnforcement = make([]string, len(c.Proxy.AllowedEgressEnforcement))
copy(result.Proxy.AllowedEgressEnforcement, c.Proxy.AllowedEgressEnforcement)
Expand Down Expand Up @@ -152,6 +170,15 @@ func (c *PlatformConfig) Validate() error {
if c.Proxy.TransparentPort < 1024 || c.Proxy.TransparentPort > 65535 {
return fmt.Errorf("proxy.transparentPort must be between 1024 and 65535")
}
if c.Proxy.TransparentInboundPort < 1024 || c.Proxy.TransparentInboundPort > 65535 {
return fmt.Errorf("proxy.transparentInboundPort must be between 1024 and 65535")
}
// The two transparent listeners are separate sockets in one container; a
// shared value would make the second bind fail at pod start, after admission
// has already succeeded. Catch it at operator startup instead.
if c.Proxy.TransparentInboundPort == c.Proxy.TransparentPort {
return fmt.Errorf("proxy.transparentInboundPort (%d) must differ from proxy.transparentPort — they are distinct listeners in the same container", c.Proxy.TransparentInboundPort)
}
// The enforce-redirect guard exempts this UID (--uid-owner) and the proxy
// container runs as it; it must be a real non-root user.
if c.Proxy.UID < 1 {
Expand All @@ -166,6 +193,14 @@ func (c *PlatformConfig) Validate() error {
default:
return fmt.Errorf("proxy.iptablesCmd %q is not a recognized backend (want one of: \"\" (auto-detect), iptables, iptables-nft, iptables-legacy)", c.Proxy.IptablesCmd)
}
if len(c.Proxy.AllowedInboundInterception) == 0 {
return fmt.Errorf("proxy.allowedInboundInterception must not be empty (set [\"reverse-proxy\"] to forbid transparent inbound, [\"transparent\"] to require it, or both to allow workload choice)")
}
for _, mode := range c.Proxy.AllowedInboundInterception {
if mode != "reverse-proxy" && mode != "transparent" {
return fmt.Errorf("proxy.allowedInboundInterception contains invalid value %q (allowed: reverse-proxy, transparent)", mode)
}
}
if len(c.Proxy.AllowedEgressEnforcement) == 0 {
return fmt.Errorf("proxy.allowedEgressEnforcement must not be empty (set [\"enforce-redirect\"] to require enforcement, [\"none\"] to disable it, or both to allow workload choice)")
}
Expand Down
32 changes: 32 additions & 0 deletions operator/internal/webhook/injector/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,38 @@ const (
EgressEnforcementNone = "none"
)

// AuthBridge's fixed (non-negotiable) listener ports in proxy-sidecar / lite.
// Unlike the reverse/forward/transparent ports these are not configurable, so
// the operator must exempt them from the transparent inbound REDIRECT by number.
// Gating AuthBridgeHealthPort in particular would put kubelet probes behind JWT
// validation and crash-loop the pod.
const (
authBridgeHealthPort = 9091 // /healthz
authBridgeStatsPort = 9093 // stats + config inspection + /reload/status
authBridgeSessionAPIPort = 9094 // session events API (consumed by abctl)
)

// Inbound interception mechanisms for the proxy-sidecar / lite paths. These
// select HOW inbound traffic reaches AuthBridge's inbound pipeline — they are
// not two levels of the same knob, but two different deployment shapes.
const (
// InboundInterceptionReverseProxy is the default: port stealing. AuthBridge
// binds the agent's original port and the agent is relocated to a free one
// via the PORT env var, so the Service needs no patching. Requires no
// privileges, but leaves the relocated port reachable directly (a pod-to-pod
// bypass of JWT validation), only covers the first declared container port,
// and silently fails for agents that hardcode their listen port.
InboundInterceptionReverseProxy = "reverse-proxy"

// InboundInterceptionTransparent installs a PREROUTING REDIRECT via
// proxy-init and lets AuthBridge recover each connection's real destination
// via SO_ORIGINAL_DST. The agent keeps its own port — no relocation, no PORT
// env var, no second port to discover — and every port it listens on is
// covered. Costs a privileged proxy-init container (NET_ADMIN) and is
// Linux-only, so it is opt-in.
InboundInterceptionTransparent = "transparent"
)

// mTLS modes for the proxy-sidecar / lite paths. Selected via the
// namespace `authbridge-runtime-config` ConfigMap's `mtls.mode` field,
// then MTLSModeDisabled. envoy-sidecar mode is incompatible with mTLS
Expand Down
97 changes: 91 additions & 6 deletions operator/internal/webhook/injector/container_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ func spireSocketDir(socketPath string) string {
// The app uses HTTP_PROXY env vars to route outbound traffic through the forward proxy.
// Inbound traffic goes through the reverse proxy.
func (b *ContainerBuilder) BuildProxySidecarContainer(spireEnabled bool) corev1.Container {
return b.BuildProxySidecarContainerWithPorts(spireEnabled, b.cfg.Images.AuthBridge, 8080, 8000, 8081)
return b.BuildProxySidecarContainerWithPorts(spireEnabled, b.cfg.Images.AuthBridge, 8080, 8081)
}

// BuildProxySidecarContainerWithPorts creates a proxy-sidecar container with dynamic ports.
Expand All @@ -231,9 +231,29 @@ func (b *ContainerBuilder) BuildProxySidecarContainer(spireEnabled bool) corev1.
// on the same ports; only the plugin set compiled into the binary differs.
//
// reverseProxyPort: where the reverse proxy listens (takes over the agent's original port)
// agentBackendPort: where the agent actually listens (moved to a free port)
// forwardProxyPort: where the forward proxy listens (HTTP_PROXY target)
func (b *ContainerBuilder) BuildProxySidecarContainerWithPorts(spireEnabled bool, image string, reverseProxyPort, agentBackendPort, forwardProxyPort int32) corev1.Container {
func (b *ContainerBuilder) BuildProxySidecarContainerWithPorts(spireEnabled bool, image string, reverseProxyPort, forwardProxyPort int32) corev1.Container {
// No agent-backend port parameter: where the agent was relocated to is
// something the sidecar learns from its ConfigMap (reverse_proxy_backend), not
// from a declared container port.
return b.buildProxySidecarContainer(spireEnabled, image, "reverse-proxy", reverseProxyPort, forwardProxyPort)
}

// BuildProxySidecarContainerTransparent builds the sidecar for
// inboundInterception "transparent": AuthBridge binds its own inbound listener
// on transparentInboundPort and iptables REDIRECTs to it, so the agent keeps its
// own port and there is no relocated backend port to declare.
//
// The declared port is named "transparent-in" rather than "reverse-proxy" so
// `kubectl describe pod` reflects which inbound shape is actually running —
// otherwise the two are indistinguishable from the pod spec.
func (b *ContainerBuilder) BuildProxySidecarContainerTransparent(spireEnabled bool, image string, transparentInboundPort, forwardProxyPort int32) corev1.Container {
// No backend port: with transparent interception the forwarding target is
// resolved per connection from SO_ORIGINAL_DST, not configured.
return b.buildProxySidecarContainer(spireEnabled, image, "transparent-in", transparentInboundPort, forwardProxyPort)
}

func (b *ContainerBuilder) buildProxySidecarContainer(spireEnabled bool, image, inboundPortName string, inboundPort, forwardProxyPort int32) corev1.Container {
volumeMounts := []corev1.VolumeMount{
{
Name: "shared-data",
Expand Down Expand Up @@ -287,8 +307,8 @@ func (b *ContainerBuilder) BuildProxySidecarContainerWithPorts(spireEnabled bool
},
Ports: []corev1.ContainerPort{
{
Name: "reverse-proxy",
ContainerPort: reverseProxyPort,
Name: inboundPortName,
ContainerPort: inboundPort,
Protocol: corev1.ProtocolTCP,
},
{
Expand Down Expand Up @@ -465,6 +485,35 @@ const mandatoryOutboundExclude = "8080"
// TRANSPARENT_PORT; the exclude args do not apply. Cluster DNS is kept
// direct by the init script reading the pod's resolv.conf nameservers.
func (b *ContainerBuilder) BuildProxyInitContainer(mode ProxyInitMode, outboundPortsExclude, inboundPortsExclude string) corev1.Container {
return b.BuildProxyInitContainerWithInbound(mode, outboundPortsExclude, inboundPortsExclude, ProxyInitInbound{})
}

// ProxyInitInbound carries the transparent-inbound additions to proxy-init's
// enforce-redirect mode. The zero value leaves inbound interception off, which
// is what BuildProxyInitContainer passes — so existing callers keep egress-only
// behavior with no change.
type ProxyInitInbound struct {
// Port is the PREROUTING REDIRECT target (INBOUND_TRANSPARENT_PORT). Zero
// means inbound interception is off.
Port int32

// SidecarPortsExclude is the comma-separated list of AuthBridge's own
// listeners to exempt from the inbound REDIRECT. The operator supplies the
// real forward-proxy port here, which may not be the script's default 8081
// when findFreePort had to move it.
SidecarPortsExclude string

// PortsExclude is the operator/user app-port exemption list
// (kagenti.io/inbound-ports-exclude), for app ports that must not be
// validated — e.g. an oauth-proxy doing its own authentication.
PortsExclude string
}

// BuildProxyInitContainerWithInbound is BuildProxyInitContainer plus the
// transparent-inbound environment. Split out rather than folded into the
// existing signature so the redirect-mode and egress-only callers stay
// untouched.
func (b *ContainerBuilder) BuildProxyInitContainerWithInbound(mode ProxyInitMode, outboundPortsExclude, inboundPortsExclude string, inbound ProxyInitInbound) corev1.Container {
var env []corev1.EnvVar
switch mode {
case ProxyInitModeEnforceRedirect:
Expand All @@ -479,10 +528,46 @@ func (b *ContainerBuilder) BuildProxyInitContainer(mode ProxyInitMode, outboundP
{Name: "PROXY_UID", Value: fmt.Sprintf("%d", b.cfg.Proxy.UID)},
{Name: "TRANSPARENT_PORT", Value: fmt.Sprintf("%d", b.cfg.Proxy.TransparentPort)},
}
// Transparent inbound (opt-in). POD_IP is REQUIRED alongside the port:
// the init script uses it as the DNAT target for the Istio ambient inbound
// path, which arrives through OUTPUT rather than PREROUTING. The script
// refuses to start without it rather than install PREROUTING-only rules
// that would wave all mesh traffic through unvalidated.
if inbound.Port > 0 {
env = append(env,
corev1.EnvVar{Name: "INBOUND_TRANSPARENT_PORT", Value: fmt.Sprintf("%d", inbound.Port)},
corev1.EnvVar{
Name: "POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"},
},
},
// POD_IPS carries BOTH families on a dual-stack pod. POD_IP alone is
// the primary address (usually v4), and the ambient DNAT target must
// match the family of the traffic — so keying only off POD_IP leaves
// the other family's HBONE delivery passing unvalidated while its
// PREROUTING rules are installed. proxy-init falls back to POD_IP
// when this is absent.
corev1.EnvVar{
Name: "POD_IPS",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIPs"},
},
},
)
if inbound.SidecarPortsExclude != "" {
env = append(env, corev1.EnvVar{Name: "SIDECAR_PORTS_EXCLUDE", Value: inbound.SidecarPortsExclude})
}
if v := buildPortExcludeValue(inbound.PortsExclude, "inbound-ports-exclude"); v != "" {
env = append(env, corev1.EnvVar{Name: "INBOUND_PORTS_EXCLUDE", Value: v})
}
}
builderLog.Info("building ProxyInit Container",
"mode", "enforce-redirect",
"proxyUID", b.cfg.Proxy.UID,
"transparentPort", b.cfg.Proxy.TransparentPort)
"transparentPort", b.cfg.Proxy.TransparentPort,
"inboundTransparentPort", inbound.Port,
"sidecarPortsExclude", inbound.SidecarPortsExclude)
case ProxyInitModeRedirect:
outboundValue := buildOutboundExcludeValue(outboundPortsExclude)
inboundValue := buildPortExcludeValue(inboundPortsExclude, "inbound-ports-exclude")
Expand Down
Loading
Loading