From 325456e9d6dca39d6128f609a55c17763922bc89 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Fri, 5 Jun 2026 18:43:59 -0400 Subject: [PATCH 1/2] feat: Make proxy-sidecar egress enforcement always-on (enforce-redirect), remove flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #406 gated proxy-sidecar egress enforcement behind proxy.egressEnforcement (default off) because the only enforcement available was fail-closed DROP, which breaks agents that ignore HTTP_PROXY. kagenti-extensions#485 adds the missing half — a transparent listener + MODE=enforce-redirect that CAPTURES bypass egress (REDIRECT to the transparent listener) instead of dropping it — so enforcement no longer breaks anything and can be always-on. This wires the operator to that: - constants: add ProxyInitModeEnforceRedirect; keep ProxyInitModeEnforceDrop as a documented no-transparent-listener fallback the operator no longer emits. - container_builder: enforce-redirect branch emits MODE / PROXY_UID / CLUSTER_CIDRS / TRANSPARENT_PORT (the new ProxyConfig.TransparentPort, default 8082, matching the authbridge listener.transparent_proxy_addr preset). - pod_mutator: the proxy-sidecar / lite branch now ALWAYS injects the enforce-redirect proxy-init (no flag check); envoy-sidecar still uses redirect. - config: REMOVE ProxyConfig.EgressEnforcement (+ its constants, validation, and the Helm value). ClusterCIDRs is now validated unconditionally (always consumed by the always-on guard), and TransparentPort gets a port-range check. - Helm values: replace egressEnforcement with transparentPort; update comments. Tests updated: builder emits enforce-redirect env incl. TRANSPARENT_PORT; proxy-sidecar always injects proxy-init (enforce-redirect); ClusterCIDRs always validated; TransparentPort validated; webhook envtest expects proxy-init. Depends on kagenti-extensions#485 (transparent listener + enforce-redirect mode) being released in the proxy-init + authbridge images. The install chart owns image versions; this PR does not pin them. Do not merge until those images ship. Preconditions for the always-on guarantee (documented, not enforced here): - UID separation — agents must not run as Proxy.UID (1337) nor hold CAP_SETUID, or they could bypass the --uid-owner exemption. Confirm via PSA restricted/SCC or an admission check. - Name-vs-IP policy — captured traffic gates on the recovered SNI/Host name (kagenti-extensions#485), which an agent controls independently of the IP, so it is reliable against a cooperative/misconfigured agent but not a hostile one. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- charts/kagenti-operator/values.yaml | 21 +++-- .../internal/webhook/config/defaults.go | 6 +- .../internal/webhook/config/types.go | 79 ++++++++----------- .../internal/webhook/config/types_test.go | 46 +++++++---- .../internal/webhook/injector/constants.go | 9 ++- .../webhook/injector/container_builder.go | 22 ++++++ .../injector/container_builder_test.go | 34 ++++++++ .../internal/webhook/injector/pod_mutator.go | 18 ++--- .../webhook/injector/pod_mutator_test.go | 57 ++++++------- .../v1alpha1/authbridge_webhook_test.go | 6 +- 10 files changed, 176 insertions(+), 122 deletions(-) diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index e26759d4..6968753e 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -188,7 +188,7 @@ featureGates: # parsers dropped) + spiffe-helper. Same listener layout # as authbridge; for size-constrained deployments. # proxy-init applies to envoy-sidecar mode (transparent redirect) and to -# proxy-sidecar mode when proxy.egressEnforcement is "enforce" (enforce-drop). +# proxy-sidecar / lite mode (always-on enforce-redirect egress capture). defaults: images: envoyProxy: ghcr.io/kagenti/kagenti-extensions/authbridge-envoy:latest @@ -203,16 +203,15 @@ defaults: uid: 1337 adminPort: 9901 inboundProxyPort: 15124 - # Proxy-sidecar fail-closed egress enforcement. "off" (default) keeps egress - # cooperative (HTTP_PROXY only); "enforce" injects a proxy-init enforce-drop - # guard so direct egress is dropped. envoy-sidecar is unaffected (it already - # enforces via transparent redirect). - egressEnforcement: "off" - # In-cluster CIDRs (pods/services/DNS) allowed direct by the enforce-drop - # guard; everything else egressing the pod is dropped. The default is - # Kind-shaped (pods 10.244/16 + services 10.96/16). OpenShift/EKS MUST - # override — e.g. OCP services 172.30.0.0/16 (outside 10/8) + pods - # 10.128.0.0/14 — or in-cluster service traffic will be dropped. + # Forward proxy's transparent listener port — the REDIRECT target for the + # always-on enforce-redirect egress guard (proxy-sidecar / lite). MUST match + # the authbridge listener.transparent_proxy_addr (default :8082). + transparentPort: 8082 + # In-cluster CIDRs (pods/services/DNS) left direct by the enforce-redirect + # guard; external TCP is REDIRECTed to the transparent listener and external + # non-TCP is dropped. The default is Kind-shaped (pods 10.244/16 + services + # 10.96/16). OpenShift/EKS MUST override — e.g. OCP services 172.30.0.0/16 + # (outside 10/8) + pods 10.128.0.0/14 — or in-cluster service traffic breaks. clusterCIDRs: - "10.0.0.0/8" diff --git a/kagenti-operator/internal/webhook/config/defaults.go b/kagenti-operator/internal/webhook/config/defaults.go index b4c7101c..4874a85c 100644 --- a/kagenti-operator/internal/webhook/config/defaults.go +++ b/kagenti-operator/internal/webhook/config/defaults.go @@ -32,9 +32,9 @@ func CompiledDefaults() *PlatformConfig { UID: 1337, InboundProxyPort: 15124, AdminPort: 9901, - // Off by default — proxy-sidecar stays cooperative (HTTP_PROXY only) - // until an operator opts in. Migrate to "enforce" in a future release. - EgressEnforcement: EgressEnforcementOff, + // Transparent listener port — must match the authbridge proxy-sidecar + // preset (listener.transparent_proxy_addr default :8082). + TransparentPort: 8082, // Kind-shaped default (pods 10.244/16 + services 10.96/16). OCP/EKS // MUST override (see ProxyConfig.ClusterCIDRs doc). ClusterCIDRs: []string{"10.0.0.0/8"}, diff --git a/kagenti-operator/internal/webhook/config/types.go b/kagenti-operator/internal/webhook/config/types.go index d99ed36b..c9960b29 100644 --- a/kagenti-operator/internal/webhook/config/types.go +++ b/kagenti-operator/internal/webhook/config/types.go @@ -41,36 +41,26 @@ type ImageConfig struct { PullPolicy corev1.PullPolicy `json:"pullPolicy" yaml:"pullPolicy"` } -// EgressEnforcement values for ProxyConfig.EgressEnforcement. -const ( - EgressEnforcementOff = "off" - EgressEnforcementEnforce = "enforce" -) - type ProxyConfig struct { Port int32 `json:"port" yaml:"port"` UID int64 `json:"uid" yaml:"uid"` InboundProxyPort int32 `json:"inboundProxyPort" yaml:"inboundProxyPort"` AdminPort int32 `json:"adminPort" yaml:"adminPort"` - // EgressEnforcement controls the proxy-sidecar fail-closed egress guard. - // "off" (default): the workload routes egress through the forward proxy - // via HTTP_PROXY only — cooperative and bypassable. - // "enforce": a proxy-init container installs the enforce-drop iptables - // guard, forcing all external egress through the forward proxy regardless - // of whether the workload honors HTTP_PROXY. - // envoy-sidecar mode is unaffected — it already enforces egress structurally - // via the transparent redirect, so this knob is consulted only on the - // proxy-sidecar / lite path. Migrate the default off->enforce in a future - // release once ClusterCIDRs sourcing is validated across distros. - EgressEnforcement string `json:"egressEnforcement" yaml:"egressEnforcement"` + // TransparentPort is the forward proxy's transparent listener port — the + // REDIRECT target the enforce-redirect proxy-init guard sends captured + // external TCP egress to. It MUST match the authbridge proxy-sidecar + // listener.transparent_proxy_addr (default :8082). + TransparentPort int32 `json:"transparentPort" yaml:"transparentPort"` // ClusterCIDRs are the in-cluster ranges (pods / services / DNS) that the - // enforce-drop guard allows direct; everything else egressing the pod is - // dropped. The default 10.0.0.0/8 is Kind-shaped (pods 10.244/16 + services - // 10.96/16). OCP/EKS MUST override this (e.g. OCP services 172.30.0.0/16, - // pods 10.128.0.0/14 — 172.30/16 is outside 10/8) or in-cluster service - // traffic will be dropped. Only used when EgressEnforcement == "enforce". + // enforce-redirect guard allows direct; external TCP is REDIRECTed to the + // transparent listener and external non-TCP is dropped. The default + // 10.0.0.0/8 is Kind-shaped (pods 10.244/16 + services 10.96/16). OCP/EKS + // MUST override this (e.g. OCP services 172.30.0.0/16, pods 10.128.0.0/14 — + // 172.30/16 is outside 10/8) or in-cluster service traffic will be dropped. + // Egress enforcement is always-on for proxy-sidecar / lite, so this is + // always consumed there; envoy-sidecar (transparent redirect) does not use it. ClusterCIDRs []string `json:"clusterCIDRs" yaml:"clusterCIDRs"` } @@ -151,32 +141,29 @@ func (c *PlatformConfig) Validate() error { if c.Proxy.AdminPort < 1024 || c.Proxy.AdminPort > 65535 { return fmt.Errorf("proxy.adminPort must be between 1024 and 65535") } - // The enforce-drop guard exempts this UID (--uid-owner) and the proxy + if c.Proxy.TransparentPort < 1024 || c.Proxy.TransparentPort > 65535 { + return fmt.Errorf("proxy.transparentPort must be between 1024 and 65535") + } + // The enforce-redirect guard exempts this UID (--uid-owner) and the proxy // container runs as it; it must be a real non-root user. if c.Proxy.UID < 1 { - return fmt.Errorf("proxy.uid must be >= 1 (got %d): the proxy must not run as root and the enforce-drop exemption keys on this UID", c.Proxy.UID) - } - switch c.Proxy.EgressEnforcement { - case "", EgressEnforcementOff, EgressEnforcementEnforce: - default: - return fmt.Errorf("proxy.egressEnforcement must be \"off\" or \"enforce\" (got %q)", c.Proxy.EgressEnforcement) - } - // When enforce is on, ClusterCIDRs drive the only in-cluster allowance in the - // enforce-drop guard. Validate them at load time so a misconfig fails fast with - // a clear message rather than: (a) an empty list silently falling back to the - // Kind-shaped 10.0.0.0/8 default in init-iptables.sh, or (b) a malformed entry - // crashing the proxy-init container under `set -e` with a cryptic iptables error. - if c.Proxy.EgressEnforcement == EgressEnforcementEnforce { - if len(c.Proxy.ClusterCIDRs) == 0 { - return fmt.Errorf("proxy.clusterCIDRs must be non-empty when proxy.egressEnforcement is %q (set the cluster's pod+service CIDRs)", EgressEnforcementEnforce) - } - // Syntactic validation only — overlapping ranges and IPv4/IPv6 mixing are - // accepted (iptables handles both, and the init script splits v4/v6 itself); - // we only reject malformed strings here. - for _, cidr := range c.Proxy.ClusterCIDRs { - if _, _, err := net.ParseCIDR(cidr); err != nil { - return fmt.Errorf("proxy.clusterCIDRs entry %q is not a valid CIDR: %w", cidr, err) - } + return fmt.Errorf("proxy.uid must be >= 1 (got %d): the proxy must not run as root and the egress-enforcement exemption keys on this UID", c.Proxy.UID) + } + // ClusterCIDRs drive the only in-cluster allowance in the enforce-redirect + // guard, which is always-on for proxy-sidecar / lite. Validate at load time so + // a misconfig fails fast with a clear message rather than: (a) an empty list + // silently falling back to the Kind-shaped 10.0.0.0/8 default in + // init-iptables.sh, or (b) a malformed entry crashing the proxy-init container + // under `set -e` with a cryptic iptables error. + if len(c.Proxy.ClusterCIDRs) == 0 { + return fmt.Errorf("proxy.clusterCIDRs must be non-empty (set the cluster's pod+service CIDRs)") + } + // Syntactic validation only — overlapping ranges and IPv4/IPv6 mixing are + // accepted (iptables handles both, and the init script splits v4/v6 itself); + // we only reject malformed strings here. + for _, cidr := range c.Proxy.ClusterCIDRs { + if _, _, err := net.ParseCIDR(cidr); err != nil { + return fmt.Errorf("proxy.clusterCIDRs entry %q is not a valid CIDR: %w", cidr, err) } } if c.Images.EnvoyProxy == "" { diff --git a/kagenti-operator/internal/webhook/config/types_test.go b/kagenti-operator/internal/webhook/config/types_test.go index 1649fd55..e27ccc5c 100644 --- a/kagenti-operator/internal/webhook/config/types_test.go +++ b/kagenti-operator/internal/webhook/config/types_test.go @@ -2,15 +2,11 @@ package config import "testing" -// When egressEnforcement is "enforce", ClusterCIDRs must be present and valid — -// an empty list (silent 10/8 fallback in the init script) or a malformed entry +// ClusterCIDRs drive the only in-cluster allowance in the always-on +// enforce-redirect guard, so they must always be present and valid — an empty +// list (silent 10/8 fallback in the init script) or a malformed entry // (init-container CrashLoop under set -e) must be rejected at config load. -func TestValidate_ClusterCIDRs_EnforceMode(t *testing.T) { - base := func() *PlatformConfig { - c := CompiledDefaults() - c.Proxy.EgressEnforcement = EgressEnforcementEnforce - return c - } +func TestValidate_ClusterCIDRs(t *testing.T) { tests := []struct { name string mutate func(*PlatformConfig) @@ -27,7 +23,7 @@ func TestValidate_ClusterCIDRs_EnforceMode(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := base() + c := CompiledDefaults() tt.mutate(c) err := c.Validate() if tt.wantErr && err == nil { @@ -40,12 +36,30 @@ func TestValidate_ClusterCIDRs_EnforceMode(t *testing.T) { } } -// When enforcement is off, ClusterCIDRs is unused and must not be validated. -func TestValidate_ClusterCIDRs_OffMode_NotValidated(t *testing.T) { - c := CompiledDefaults() - c.Proxy.EgressEnforcement = EgressEnforcementOff - c.Proxy.ClusterCIDRs = []string{"garbage"} - if err := c.Validate(); err != nil { - t.Errorf("off mode must not validate ClusterCIDRs, got: %v", err) +// TransparentPort is the REDIRECT target for the enforce-redirect guard; it must +// be a valid, non-privileged port (the proxy binds it as a non-root user). +func TestValidate_TransparentPort(t *testing.T) { + tests := []struct { + name string + port int32 + wantErr bool + }{ + {"default 8082 ok", 8082, false}, + {"zero rejected", 0, true}, + {"privileged rejected", 80, true}, + {"too large rejected", 70000, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := CompiledDefaults() + c.Proxy.TransparentPort = tt.port + err := c.Validate() + if tt.wantErr && err == nil { + t.Errorf("expected validation error, got nil") + } + if !tt.wantErr && err != nil { + t.Errorf("unexpected validation error: %v", err) + } + }) } } diff --git a/kagenti-operator/internal/webhook/injector/constants.go b/kagenti-operator/internal/webhook/injector/constants.go index 96332d80..30709bf6 100644 --- a/kagenti-operator/internal/webhook/injector/constants.go +++ b/kagenti-operator/internal/webhook/injector/constants.go @@ -50,8 +50,15 @@ const ( // ProxyInitModeRedirect transparently REDIRECTs pod traffic to the Envoy // listeners (envoy-sidecar mode). ProxyInitModeRedirect ProxyInitMode = "redirect" + // ProxyInitModeEnforceRedirect installs the fail-closed egress guard that + // REDIRECTs external TCP bypassing the forward proxy to AuthBridge's + // transparent listener (captured, not dropped) and DROPs non-TCP external + // egress. Always-on for proxy-sidecar / lite. + ProxyInitModeEnforceRedirect ProxyInitMode = "enforce-redirect" // ProxyInitModeEnforceDrop installs the fail-closed egress guard that DROPs - // any egress bypassing the forward proxy (proxy-sidecar egress enforcement). + // any egress bypassing the forward proxy. Predates enforce-redirect; the + // init image retains it as a no-transparent-listener fallback, but the + // operator no longer emits it. ProxyInitModeEnforceDrop ProxyInitMode = "enforce-drop" ) diff --git a/kagenti-operator/internal/webhook/injector/container_builder.go b/kagenti-operator/internal/webhook/injector/container_builder.go index 7d8b2c17..0c0b4fa3 100644 --- a/kagenti-operator/internal/webhook/injector/container_builder.go +++ b/kagenti-operator/internal/webhook/injector/container_builder.go @@ -459,12 +459,34 @@ const mandatoryOutboundExclude = "8080" // Envoy listeners. outboundPortsExclude / inboundPortsExclude (from the // kagenti.io/outbound-ports-exclude and kagenti.io/inbound-ports-exclude // pod annotations; mandatory 8080 always included outbound) tune it. +// - "enforce-redirect" (proxy-sidecar): a fail-closed egress guard that +// REDIRECTs external TCP bypassing the forward proxy to the transparent +// listener (TRANSPARENT_PORT) and DROPs non-TCP. Driven by PROXY_UID + +// CLUSTER_CIDRS + TRANSPARENT_PORT; the exclude args do not apply. // - "enforce-drop" (proxy-sidecar): a fail-closed egress guard that DROPs any // egress bypassing the forward proxy. Driven by PROXY_UID + CLUSTER_CIDRS; // the exclude args do not apply (the script ignores them in this mode). func (b *ContainerBuilder) BuildProxyInitContainer(mode ProxyInitMode, outboundPortsExclude, inboundPortsExclude string) corev1.Container { var env []corev1.EnvVar switch mode { + case ProxyInitModeEnforceRedirect: + // PROXY_UID is exempted (its egress is not redirected) and MUST match the + // proxy container's RunAsUser (both derive from b.cfg.Proxy.UID). + // CLUSTER_CIDRS are allowed direct; external TCP is REDIRECTed to + // TRANSPARENT_PORT (the proxy's transparent listener), external non-TCP is + // dropped. The redirect-only exclude vars are unused in this mode. + clusterCIDRs := strings.Join(b.cfg.Proxy.ClusterCIDRs, ",") + env = []corev1.EnvVar{ + {Name: "MODE", Value: string(ProxyInitModeEnforceRedirect)}, + {Name: "PROXY_UID", Value: fmt.Sprintf("%d", b.cfg.Proxy.UID)}, + {Name: "CLUSTER_CIDRS", Value: clusterCIDRs}, + {Name: "TRANSPARENT_PORT", Value: fmt.Sprintf("%d", b.cfg.Proxy.TransparentPort)}, + } + builderLog.Info("building ProxyInit Container", + "mode", "enforce-redirect", + "proxyUID", b.cfg.Proxy.UID, + "transparentPort", b.cfg.Proxy.TransparentPort, + "clusterCIDRs", clusterCIDRs) case ProxyInitModeEnforceDrop: // PROXY_UID is exempted from the DROP and MUST match the proxy // container's RunAsUser (both derive from b.cfg.Proxy.UID). CLUSTER_CIDRS diff --git a/kagenti-operator/internal/webhook/injector/container_builder_test.go b/kagenti-operator/internal/webhook/injector/container_builder_test.go index 0ff46d7d..65653a9d 100644 --- a/kagenti-operator/internal/webhook/injector/container_builder_test.go +++ b/kagenti-operator/internal/webhook/injector/container_builder_test.go @@ -335,6 +335,40 @@ func TestBuildProxyInitContainer_EnforceDrop(t *testing.T) { } } +// enforce-redirect mode emits MODE / PROXY_UID / CLUSTER_CIDRS / TRANSPARENT_PORT +// and none of the redirect-only vars (POD_IP, OUTBOUND_PORTS_EXCLUDE). +func TestBuildProxyInitContainer_EnforceRedirect(t *testing.T) { + cfg := config.CompiledDefaults() + builder := NewContainerBuilder(cfg) + container := builder.BuildProxyInitContainer("enforce-redirect", "", "") + + got := map[string]string{} + for _, e := range container.Env { + if e.ValueFrom != nil { + t.Errorf("enforce-redirect env %q must be a literal, not ValueFrom", e.Name) + } + got[e.Name] = e.Value + } + if got["MODE"] != "enforce-redirect" { + t.Errorf("MODE = %q, want enforce-redirect", got["MODE"]) + } + if want := strconv.FormatInt(cfg.Proxy.UID, 10); got["PROXY_UID"] != want { + t.Errorf("PROXY_UID = %q, want %q", got["PROXY_UID"], want) + } + if want := strings.Join(cfg.Proxy.ClusterCIDRs, ","); got["CLUSTER_CIDRS"] != want { + t.Errorf("CLUSTER_CIDRS = %q, want %q", got["CLUSTER_CIDRS"], want) + } + if want := strconv.FormatInt(int64(cfg.Proxy.TransparentPort), 10); got["TRANSPARENT_PORT"] != want { + t.Errorf("TRANSPARENT_PORT = %q, want %q", got["TRANSPARENT_PORT"], want) + } + if _, ok := got["POD_IP"]; ok { + t.Error("enforce-redirect must not set POD_IP") + } + if _, ok := got["OUTBOUND_PORTS_EXCLUDE"]; ok { + t.Error("enforce-redirect must not set OUTBOUND_PORTS_EXCLUDE") + } +} + // An unknown mode must fail closed: BuildProxyInitContainer returns a // zero-value container (no name/image/env) rather than silently degrading to // redirect, which would ship a proxy-init with no egress guard (fail-open). diff --git a/kagenti-operator/internal/webhook/injector/pod_mutator.go b/kagenti-operator/internal/webhook/injector/pod_mutator.go index a4725c03..53ff0bab 100644 --- a/kagenti-operator/internal/webhook/injector/pod_mutator.go +++ b/kagenti-operator/internal/webhook/injector/pod_mutator.go @@ -525,18 +525,18 @@ func (m *PodMutator) InjectAuthBridge(ctx context.Context, podSpec *corev1.PodSp injectHTTPProxyEnv(c, forwardProxyPort) } - // Fail-closed egress enforcement (opt-in via proxy.egressEnforcement, - // default "off"). HTTP_PROXY above is cooperative — an app that ignores - // it egresses directly. When enforcement is on, inject the proxy-init - // enforce-drop guard so any direct egress is dropped. envoy-sidecar - // already enforces structurally via transparent redirect, so this is the + // Fail-closed egress enforcement (always-on for proxy-sidecar / lite). + // HTTP_PROXY above is cooperative — an app that ignores it egresses + // directly. The enforce-redirect proxy-init guard transparently REDIRECTs + // any bypass egress to the forward proxy's transparent listener, so it is + // captured rather than dropped and nothing breaks. envoy-sidecar enforces + // structurally via its own transparent redirect, so this is the // proxy-sidecar / lite path only. The exempted PROXY_UID equals the proxy // container's RunAsUser (both b.cfg.Proxy.UID). - if currentConfig.Proxy.EgressEnforcement == config.EgressEnforcementEnforce && - !containerExists(podSpec.InitContainers, ProxyInitContainerName) { + if !containerExists(podSpec.InitContainers, ProxyInitContainerName) { podSpec.InitContainers = append(podSpec.InitContainers, - builder.BuildProxyInitContainer(ProxyInitModeEnforceDrop, "", "")) - mutatorLog.Info("proxy-sidecar egress enforcement enabled (enforce-drop)", + builder.BuildProxyInitContainer(ProxyInitModeEnforceRedirect, "", "")) + mutatorLog.Info("proxy-sidecar egress enforcement enabled (enforce-redirect)", "namespace", namespace, "crName", crName) } diff --git a/kagenti-operator/internal/webhook/injector/pod_mutator_test.go b/kagenti-operator/internal/webhook/injector/pod_mutator_test.go index 2204e7bf..8abd0c4d 100644 --- a/kagenti-operator/internal/webhook/injector/pod_mutator_test.go +++ b/kagenti-operator/internal/webhook/injector/pod_mutator_test.go @@ -167,16 +167,16 @@ func TestInjectAuthBridge_NoAgentRuntime_InjectsWithDefaults(t *testing.T) { t.Fatal("expected InjectAuthBridge to return true with defaults-only config") } - // Default mode is proxy-sidecar — expect authbridge-proxy container, - // no envoy-proxy / proxy-init / standalone spiffe-helper. + // Default mode is proxy-sidecar — expect authbridge-proxy container and the + // always-on enforce-redirect proxy-init guard; no envoy-proxy. if !containerExists(podSpec.Containers, AuthBridgeProxyContainerName) { t.Errorf("expected %s container to be injected", AuthBridgeProxyContainerName) } if containerExists(podSpec.Containers, EnvoyProxyContainerName) { t.Errorf("unexpected %s container in proxy-sidecar mode", EnvoyProxyContainerName) } - if containerExists(podSpec.InitContainers, ProxyInitContainerName) { - t.Errorf("unexpected %s init container in proxy-sidecar mode", ProxyInitContainerName) + if !containerExists(podSpec.InitContainers, ProxyInitContainerName) { + t.Errorf("expected %s init container in proxy-sidecar mode (always-on enforce-redirect)", ProxyInitContainerName) } } @@ -781,9 +781,9 @@ func TestInjectAuthBridge_WaypointMode_SkipsInjection(t *testing.T) { } } -// Egress enforcement for proxy-sidecar is gated by proxy.egressEnforcement -// (default "off"). When "enforce", a proxy-init container is injected in -// enforce-drop mode; envoy-sidecar is unaffected (tested elsewhere). +// Egress enforcement is always-on for proxy-sidecar: a proxy-init container is +// always injected in enforce-redirect mode; envoy-sidecar is unaffected (it +// uses redirect mode, tested elsewhere). func TestInjectAuthBridge_ProxySidecar_EgressEnforcement(t *testing.T) { ctx := context.Background() labels := map[string]string{KagentiTypeLabel: KagentiTypeAgent} @@ -802,48 +802,35 @@ func TestInjectAuthBridge_ProxySidecar_EgressEnforcement(t *testing.T) { return nil } - t.Run("enforce injects proxy-init in enforce-drop mode", func(t *testing.T) { + t.Run("always injects proxy-init in enforce-redirect mode", func(t *testing.T) { m := newTestMutator(newAgentRuntimeWithMode("team1", "my-agent", ModeProxySidecar)) - cfg := config.CompiledDefaults() - cfg.Proxy.EgressEnforcement = "enforce" - m.GetPlatformConfig = func() *config.PlatformConfig { return cfg } - spec := makePod() if _, err := m.InjectAuthBridge(ctx, spec, "team1", "my-agent", labels, nil); err != nil { t.Fatalf("unexpected error: %v", err) } ic := findProxyInit(spec) if ic == nil { - t.Fatal("proxy-init should be injected when egressEnforcement=enforce") + t.Fatal("proxy-init should always be injected for proxy-sidecar") } - var mode string + var mode, transparentPort string for _, e := range ic.Env { - if e.Name == "MODE" { + switch e.Name { + case "MODE": mode = e.Value + case "TRANSPARENT_PORT": + transparentPort = e.Value } } - if mode != "enforce-drop" { - t.Errorf("proxy-init MODE = %q, want enforce-drop", mode) + if mode != "enforce-redirect" { + t.Errorf("proxy-init MODE = %q, want enforce-redirect", mode) } - }) - - t.Run("default off does not inject proxy-init", func(t *testing.T) { - m := newTestMutator(newAgentRuntimeWithMode("team1", "my-agent", ModeProxySidecar)) - spec := makePod() - if _, err := m.InjectAuthBridge(ctx, spec, "team1", "my-agent", labels, nil); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if findProxyInit(spec) != nil { - t.Error("proxy-init must not be injected when egressEnforcement is off (default)") + if transparentPort == "" { + t.Error("enforce-redirect must set TRANSPARENT_PORT") } }) t.Run("does not duplicate an existing proxy-init", func(t *testing.T) { m := newTestMutator(newAgentRuntimeWithMode("team1", "my-agent", ModeProxySidecar)) - cfg := config.CompiledDefaults() - cfg.Proxy.EgressEnforcement = "enforce" - m.GetPlatformConfig = func() *config.PlatformConfig { return cfg } - spec := makePod() spec.InitContainers = []corev1.Container{{Name: ProxyInitContainerName, Image: "preexisting"}} if _, err := m.InjectAuthBridge(ctx, spec, "team1", "my-agent", labels, nil); err != nil { @@ -897,12 +884,16 @@ func TestInjectAuthBridge_ProxySidecarMode_InjectsCorrectly(t *testing.T) { t.Error("authbridge-proxy container not found") } - // Should NOT have proxy-init (no iptables in proxy-sidecar mode) + // Should have the always-on enforce-redirect proxy-init guard. + proxyInitFound := false for _, c := range podSpec.InitContainers { if c.Name == ProxyInitContainerName { - t.Error("proxy-init should not be injected in proxy-sidecar mode") + proxyInitFound = true } } + if !proxyInitFound { + t.Error("proxy-init (enforce-redirect) should be injected in proxy-sidecar mode") + } // Should NOT have envoy-proxy container for _, c := range podSpec.Containers { diff --git a/kagenti-operator/internal/webhook/v1alpha1/authbridge_webhook_test.go b/kagenti-operator/internal/webhook/v1alpha1/authbridge_webhook_test.go index 12dd10f9..4538de75 100644 --- a/kagenti-operator/internal/webhook/v1alpha1/authbridge_webhook_test.go +++ b/kagenti-operator/internal/webhook/v1alpha1/authbridge_webhook_test.go @@ -188,11 +188,11 @@ var _ = Describe("AuthBridge Pod Webhook", func() { err = k8sClient.Get(ctx, client.ObjectKeyFromObject(pod), pod) Expect(err).NotTo(HaveOccurred()) - // Default mode is proxy-sidecar — expect authbridge-proxy, no - // envoy-proxy or proxy-init. + // Default mode is proxy-sidecar — expect authbridge-proxy and the + // always-on enforce-redirect proxy-init guard; no envoy-proxy. Expect(containerNames(pod.Spec.Containers)).To(ContainElement(injector.AuthBridgeProxyContainerName)) Expect(containerNames(pod.Spec.Containers)).NotTo(ContainElement(injector.EnvoyProxyContainerName)) - Expect(initContainerNames(pod.Spec.InitContainers)).NotTo(ContainElement(injector.ProxyInitContainerName)) + Expect(initContainerNames(pod.Spec.InitContainers)).To(ContainElement(injector.ProxyInitContainerName)) }) }) From fb1414569095ba515d2a75df52b66d4ac4340631 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Fri, 5 Jun 2026 18:59:13 -0400 Subject: [PATCH 2/2] refactor: drop the obsolete ProxyInitModeEnforceDrop The operator now only ever emits enforce-redirect, and kagenti-extensions removes the enforce-drop MODE from the init script (superseded by the transparent listener capturing bypass egress instead of dropping it). Remove the dead ProxyInitModeEnforceDrop constant, its BuildProxyInitContainer branch, and its test; update stale comments. The builder now handles redirect | enforce-redirect, with the fail-closed default for unknown modes. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- .../internal/webhook/injector/constants.go | 5 --- .../webhook/injector/container_builder.go | 20 +---------- .../injector/container_builder_test.go | 35 ++----------------- 3 files changed, 3 insertions(+), 57 deletions(-) diff --git a/kagenti-operator/internal/webhook/injector/constants.go b/kagenti-operator/internal/webhook/injector/constants.go index 30709bf6..b740a091 100644 --- a/kagenti-operator/internal/webhook/injector/constants.go +++ b/kagenti-operator/internal/webhook/injector/constants.go @@ -55,11 +55,6 @@ const ( // transparent listener (captured, not dropped) and DROPs non-TCP external // egress. Always-on for proxy-sidecar / lite. ProxyInitModeEnforceRedirect ProxyInitMode = "enforce-redirect" - // ProxyInitModeEnforceDrop installs the fail-closed egress guard that DROPs - // any egress bypassing the forward proxy. Predates enforce-redirect; the - // init image retains it as a no-transparent-listener fallback, but the - // operator no longer emits it. - ProxyInitModeEnforceDrop ProxyInitMode = "enforce-drop" ) // mTLS modes for the proxy-sidecar / lite paths. Selected per workload diff --git a/kagenti-operator/internal/webhook/injector/container_builder.go b/kagenti-operator/internal/webhook/injector/container_builder.go index 0c0b4fa3..850c26fa 100644 --- a/kagenti-operator/internal/webhook/injector/container_builder.go +++ b/kagenti-operator/internal/webhook/injector/container_builder.go @@ -300,7 +300,7 @@ func (b *ContainerBuilder) BuildProxySidecarContainerWithPorts(spireEnabled bool Resources: b.cfg.Resources.AuthBridge, SecurityContext: &corev1.SecurityContext{ // Run as the dedicated proxy UID (default 1337), the SAME value the - // proxy-init enforce-drop guard exempts via `--uid-owner $PROXY_UID`. + // proxy-init enforce-redirect guard exempts via `--uid-owner $PROXY_UID`. // Deriving both from b.cfg.Proxy.UID keeps the exempted UID in lockstep // with the process it exempts (a hardcoded literal could drift and // silently break the proxy's own egress). Matches the envoy builder. @@ -463,9 +463,6 @@ const mandatoryOutboundExclude = "8080" // REDIRECTs external TCP bypassing the forward proxy to the transparent // listener (TRANSPARENT_PORT) and DROPs non-TCP. Driven by PROXY_UID + // CLUSTER_CIDRS + TRANSPARENT_PORT; the exclude args do not apply. -// - "enforce-drop" (proxy-sidecar): a fail-closed egress guard that DROPs any -// egress bypassing the forward proxy. Driven by PROXY_UID + CLUSTER_CIDRS; -// the exclude args do not apply (the script ignores them in this mode). func (b *ContainerBuilder) BuildProxyInitContainer(mode ProxyInitMode, outboundPortsExclude, inboundPortsExclude string) corev1.Container { var env []corev1.EnvVar switch mode { @@ -487,21 +484,6 @@ func (b *ContainerBuilder) BuildProxyInitContainer(mode ProxyInitMode, outboundP "proxyUID", b.cfg.Proxy.UID, "transparentPort", b.cfg.Proxy.TransparentPort, "clusterCIDRs", clusterCIDRs) - case ProxyInitModeEnforceDrop: - // PROXY_UID is exempted from the DROP and MUST match the proxy - // container's RunAsUser (both derive from b.cfg.Proxy.UID). CLUSTER_CIDRS - // are allowed direct; everything else egressing the pod is dropped. - // POD_IP and the redirect-only exclude vars are unused in this mode. - clusterCIDRs := strings.Join(b.cfg.Proxy.ClusterCIDRs, ",") - env = []corev1.EnvVar{ - {Name: "MODE", Value: string(ProxyInitModeEnforceDrop)}, - {Name: "PROXY_UID", Value: fmt.Sprintf("%d", b.cfg.Proxy.UID)}, - {Name: "CLUSTER_CIDRS", Value: clusterCIDRs}, - } - builderLog.Info("building ProxyInit Container", - "mode", "enforce-drop", - "proxyUID", b.cfg.Proxy.UID, - "clusterCIDRs", clusterCIDRs) case ProxyInitModeRedirect: outboundValue := buildOutboundExcludeValue(outboundPortsExclude) inboundValue := buildPortExcludeValue(inboundPortsExclude, "inbound-ports-exclude") diff --git a/kagenti-operator/internal/webhook/injector/container_builder_test.go b/kagenti-operator/internal/webhook/injector/container_builder_test.go index 65653a9d..6f67bcce 100644 --- a/kagenti-operator/internal/webhook/injector/container_builder_test.go +++ b/kagenti-operator/internal/webhook/injector/container_builder_test.go @@ -285,8 +285,8 @@ func TestBuildProxyInitContainer_WithBothExcludes(t *testing.T) { } // The proxy-sidecar container must run as the configured Proxy.UID — the same -// value the enforce-drop guard exempts via --uid-owner. If these drift, the -// proxy's own egress would be dropped (or the agent would share the exempt UID). +// value the enforce-redirect guard exempts via --uid-owner. If these drift, the +// proxy's own egress would be redirected back (or the agent would share the exempt UID). func TestBuildProxySidecarContainer_RunAsProxyUID(t *testing.T) { cfg := config.CompiledDefaults() builder := NewContainerBuilder(cfg) @@ -304,37 +304,6 @@ func TestBuildProxySidecarContainer_RunAsProxyUID(t *testing.T) { } } -// enforce-drop mode emits MODE / PROXY_UID / CLUSTER_CIDRS and none of the -// redirect-only vars (POD_IP, OUTBOUND_PORTS_EXCLUDE). -func TestBuildProxyInitContainer_EnforceDrop(t *testing.T) { - cfg := config.CompiledDefaults() - builder := NewContainerBuilder(cfg) - container := builder.BuildProxyInitContainer("enforce-drop", "", "") - - got := map[string]string{} - for _, e := range container.Env { - if e.ValueFrom != nil { - t.Errorf("enforce-drop env %q must be a literal, not ValueFrom", e.Name) - } - got[e.Name] = e.Value - } - if got["MODE"] != "enforce-drop" { - t.Errorf("MODE = %q, want enforce-drop", got["MODE"]) - } - if want := strconv.FormatInt(cfg.Proxy.UID, 10); got["PROXY_UID"] != want { - t.Errorf("PROXY_UID = %q, want %q", got["PROXY_UID"], want) - } - if want := strings.Join(cfg.Proxy.ClusterCIDRs, ","); got["CLUSTER_CIDRS"] != want { - t.Errorf("CLUSTER_CIDRS = %q, want %q", got["CLUSTER_CIDRS"], want) - } - if _, ok := got["POD_IP"]; ok { - t.Error("enforce-drop must not set POD_IP") - } - if _, ok := got["OUTBOUND_PORTS_EXCLUDE"]; ok { - t.Error("enforce-drop must not set OUTBOUND_PORTS_EXCLUDE") - } -} - // enforce-redirect mode emits MODE / PROXY_UID / CLUSTER_CIDRS / TRANSPARENT_PORT // and none of the redirect-only vars (POD_IP, OUTBOUND_PORTS_EXCLUDE). func TestBuildProxyInitContainer_EnforceRedirect(t *testing.T) {