From 5b7b50c0694d4f8eda492eb06621e6f02e38554e Mon Sep 17 00:00:00 2001 From: "K.J. Valencik" Date: Mon, 13 Jul 2026 13:57:04 -0400 Subject: [PATCH] feat(main): add --watch-namespace to scope the manager cache The manager's default cluster-wide cache holds every Pod, PVC, Service, Job, and PDB in the cluster; on large clusters this OOM-kills the manager against the chart's default 128Mi limit. Add a --watch-namespace flag (defaulting to $WATCH_NAMESPACE, per the operator-sdk convention) taking a comma-separated namespace list, wired into the manager's Cache.DefaultNamespaces. Empty/unset preserves the current watch-everything behavior. The chart exposes it as manager.watchNamespaces, rendered onto the manager container as the WATCH_NAMESPACE env var. RBAC is intentionally untouched: the ClusterRole continues to authorize the scoped watches. Signed-off-by: K.J. Valencik --- .../etcd-operator/templates/deployment.yaml | 4 ++ charts/etcd-operator/values.yaml | 2 + docs/installation.md | 3 +- main.go | 25 ++++++++ main_test.go | 58 +++++++++++++++++++ 5 files changed, 91 insertions(+), 1 deletion(-) diff --git a/charts/etcd-operator/templates/deployment.yaml b/charts/etcd-operator/templates/deployment.yaml index feb4eecd..fcdb545d 100644 --- a/charts/etcd-operator/templates/deployment.yaml +++ b/charts/etcd-operator/templates/deployment.yaml @@ -59,6 +59,10 @@ spec: # the tag is always v. - name: ETCD_IMAGE_REPOSITORY value: {{ .Values.etcdImage.repository | quote }} + {{- with .Values.manager.watchNamespaces }} + - name: WATCH_NAMESPACE + value: {{ if kindIs "slice" . }}{{ join "," . | quote }}{{ else }}{{ . | quote }}{{ end }} + {{- end }} livenessProbe: httpGet: path: /healthz diff --git a/charts/etcd-operator/values.yaml b/charts/etcd-operator/values.yaml index 806ca27d..04233fbb 100644 --- a/charts/etcd-operator/values.yaml +++ b/charts/etcd-operator/values.yaml @@ -83,6 +83,8 @@ tolerations: [] affinity: {} manager: + # -- Namespaces the manager watches (WATCH_NAMESPACE env var). Empty = all. + watchNamespaces: [] # -- Resource requests/limits for the manager container. resources: limits: diff --git a/docs/installation.md b/docs/installation.md index bcf2a932..d8c6ab4c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -97,6 +97,7 @@ Common values (`--set key=value`, or a `-f my-values.yaml`): | `metrics.serviceMonitor.enabled` | `false` | Create a prometheus-operator `ServiceMonitor` for the metrics endpoint (needs the `monitoring.coreos.com` CRDs and `kubeRbacProxy.enabled`). | | `crds.enabled` / `crds.keep` | `true` / `true` | Render the CRDs with the release / annotate them so uninstall keeps them. | | `manager.resources` | 10m/64Mi → 500m/128Mi | Manager container requests/limits. | +| `manager.watchNamespaces` | `[]` | Namespaces the manager watches. Empty = all (see [RBAC](#rbac)). | | `imagePullSecrets` | `[]` | Pull secrets for the **operator's own** image (private registry mirror). | | `etcdImage.repository` | `quay.io/coreos/etcd` | Operator-wide default **etcd** image repo for member Pods (always wired into `ETCD_IMAGE_REPOSITORY`). Repoint at an air-gapped mirror once; the tag is always `v`. | @@ -280,7 +281,7 @@ All pinned in `go.mod`, `Dockerfile`, and `Makefile`. The operator runs as a ClusterRole — it needs to watch `EtcdCluster` and `EtcdMember` across all namespaces, plus create/delete the per-member Pods, PVCs, and Services in each user namespace. The rules are generated from the `+kubebuilder:rbac` markers (by `make manifests`) into `charts/etcd-operator/files/manager-role-rules.yaml` and pulled into the chart's templated ClusterRole — don't hand-edit; edit the markers and regenerate. -Single-namespace scoping is not currently exposed: `main.go` does not wire a namespace flag into the manager's `Cache.DefaultNamespaces`, so the manager always watches all namespaces. Limiting RBAC alone (ClusterRole → Role) is not sufficient — the manager will still attempt list/watch across the cluster and the API server will deny it. Scoped deployment is a follow-up. +By default the manager watches all namespaces. To scope it, set `--watch-namespace` / `WATCH_NAMESPACE` (comma-separated), or `--set 'manager.watchNamespaces={tenant-a,tenant-b}'` via the chart. Scoping bounds cache memory on large clusters. RBAC must still permit the watches; the chart's ClusterRole does. Switching to namespaced `Role`s also requires `kubeRbacProxy.enabled=false` — the proxy's TokenReview/SubjectAccessReview permissions are cluster-scoped. ## Networking diff --git a/main.go b/main.go index 3d6450ba..3e22a4cf 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -115,6 +116,7 @@ func main() { var clusterDomain string var operatorImage string var etcdImageRepository string + var watchNamespace string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, @@ -139,6 +141,8 @@ func main() { "air-gapped deployments at a mirror once; the tag is always v. "+ "Defaults to $ETCD_IMAGE_REPOSITORY; when empty the built-in "+ "quay.io/coreos/etcd is used.") + flag.StringVar(&watchNamespace, "watch-namespace", os.Getenv("WATCH_NAMESPACE"), + "Comma-separated namespaces to watch. Defaults to $WATCH_NAMESPACE; empty means all.") opts := zap.Options{ Development: true, } @@ -156,8 +160,14 @@ func main() { os.Exit(1) } + cacheOptions := watchNamespaceCacheOptions(watchNamespace) + if len(cacheOptions.DefaultNamespaces) > 0 { + setupLog.Info("scoping cache and watches to namespaces", "watchNamespace", watchNamespace) + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + Cache: cacheOptions, Metrics: metricsserver.Options{BindAddress: metricsAddr}, WebhookServer: webhook.NewServer(webhook.Options{Port: 9443}), HealthProbeBindAddress: probeAddr, @@ -289,6 +299,21 @@ func discoverClusterDomain(path string) string { return "" } +// watchNamespaceCacheOptions scopes the manager cache to a comma-separated +// namespace list. Empty input means all namespaces (zero cache.Options). +func watchNamespaceCacheOptions(watchNamespace string) cache.Options { + namespaces := map[string]cache.Config{} + for _, ns := range strings.Split(watchNamespace, ",") { + if ns = strings.TrimSpace(ns); ns != "" { + namespaces[ns] = cache.Config{} + } + } + if len(namespaces) == 0 { + return cache.Options{} + } + return cache.Options{DefaultNamespaces: namespaces} +} + // detectCertManager probes the apiserver's discovery API for the // cert-manager.io/v1 Certificate kind. We detect once at startup rather // than lazily on first use because: diff --git a/main_test.go b/main_test.go index aab360e6..0a741906 100644 --- a/main_test.go +++ b/main_test.go @@ -13,7 +13,10 @@ package main import ( "os" "path/filepath" + "reflect" "testing" + + "sigs.k8s.io/controller-runtime/pkg/cache" ) func TestDiscoverClusterDomain(t *testing.T) { @@ -106,6 +109,61 @@ func TestDiscoverClusterDomain_MissingFile(t *testing.T) { } } +func TestWatchNamespaceCacheOptions(t *testing.T) { + cases := []struct { + name string + raw string + want map[string]cache.Config + }{ + { + name: "empty means all namespaces", + raw: "", + want: nil, + }, + { + name: "whitespace only means all namespaces", + raw: " ", + want: nil, + }, + { + name: "separators only means all namespaces", + raw: " , ,, ", + want: nil, + }, + { + name: "single namespace", + raw: "tenant-a", + want: map[string]cache.Config{"tenant-a": {}}, + }, + { + name: "multiple namespaces", + raw: "tenant-a,tenant-b", + want: map[string]cache.Config{"tenant-a": {}, "tenant-b": {}}, + }, + { + name: "whitespace and empty entries are dropped", + raw: " tenant-a , ,tenant-b, ", + want: map[string]cache.Config{"tenant-a": {}, "tenant-b": {}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := watchNamespaceCacheOptions(tc.raw) + if tc.want == nil { + // Zero value keeps the unset path identical to the old config. + if !reflect.DeepEqual(got, cache.Options{}) { + t.Fatalf("watchNamespaceCacheOptions(%q) = %+v; want zero cache.Options", tc.raw, got) + } + return + } + if !reflect.DeepEqual(got.DefaultNamespaces, tc.want) { + t.Fatalf("watchNamespaceCacheOptions(%q).DefaultNamespaces = %+v; want %+v", tc.raw, got.DefaultNamespaces, tc.want) + } + }) + } +} + func TestOperatorImageError(t *testing.T) { if err := operatorImageError(placeholderOperatorImage); err == nil { t.Errorf("operatorImageError(%q) = nil, want error (placeholder must be rejected)", placeholderOperatorImage)