Fix: Do not cache native resources created without CommonLabels (#1818)

* fix: Limit cache for k8s native resources

* feat: Disable caching of ConfigMaps and Secrets

feat: Toggle caching of ConfigMaps and Secrets with CommonLabels

* chore: Move sharding log out of main (cyclomatic lint)

* chore: Update commments

* Refactor: Re-use new util func to create cacheLabelConfig for default setups

* fix: Avoid labels.Everything returned from getLabelSelectors

* feat: Add a way to disable the cache limitations for backwards compatibility

* chore: Make cache improvements opt-in

* chore: Update env var name and "watchLabeledReferencesOnly" description in Helm values

* refactor: switch to `ENFORCE_CACHE_LABELS` env var

---------

Co-authored-by: Dominik Süß <[email protected]>

Author: Baarsgaard

deploy/helm/grafana-operator/README.md

@@ -68,6 +68,7 @@ It's easier to just manage this configuration outside of the operator.
 | dashboard.annotations | object | `{}` | Annotations to add to the Grafana dashboard ConfigMap |
 | dashboard.enabled | bool | `false` | Whether to create a ConfigMap containing a dashboard monitoring the operator metrics. Consider enabling this if you are enabling the ServiceMonitor. Optionally, a GrafanaDashboard CR can be manually created pointing to the Grafana.com dashboard ID 22785 https://grafana.com/grafana/dashboards/22785-grafana-operator/ The Grafana.com dashboard is maintained by the community and does not necessarily match the JSON definition in this repository. |
 | dashboard.labels | object | `{}` | Labels to add to the Grafana dashboard ConfigMap |
+| enforceCacheLabels | string | `"off"` | Sets the `ENFORCE_CACHE_LABELS` environment variable, Enables the enforcment of cache labels, reducing memory usage significantly. Valid values are "off","safe" and "all". When using `safe` mode, ConfigMaps and Secrets are excluded from caching. When using `all` mode, ConfigMaps and Secrets are cached and require the `app.kubernetes.io/managed-by: grafana-operator` label. |
 | env | list | `[]` | Additional environment variables |
 | extraObjects | list | `[]` | Array of extra K8s objects to deploy |
 | fullnameOverride | string | `""` | Overrides the fully qualified app name. |

deploy/helm/grafana-operator/templates/deployment.yaml

@@ -60,6 +60,12 @@ spec:
               {{ else }}
               value: {{quote .Values.watchLabelSelectors }}
               {{- end }}
+            - name: ENFORCE_CACHE_LABELS
+              {{- if .Values.enforceCacheLabels }}
+              value: {{quote .Values.enforceCacheLabels }}
+              {{ else }}
+              value: "off"
+              {{- end }}
             - name: CLUSTER_DOMAIN
               {{- if and .Values.clusterDomain (eq .Values.clusterDomain "") }}
               value: ""

deploy/helm/grafana-operator/values.yaml

@@ -24,6 +24,13 @@ watchNamespaceSelector: ""
 watchLabelSelectors: ""
 # watchLabelSelectors: "partition in (customerA, customerB),environment!=qa"
 
+# -- Sets the `ENFORCE_CACHE_LABELS` environment variable,
+# Enables the enforcment of cache labels, reducing memory usage significantly.
+# Valid values are "off","safe" and "all".
+# When using `safe` mode, ConfigMaps and Secrets are excluded from caching.
+# When using `all` mode, ConfigMaps and Secrets are cached and require the `app.kubernetes.io/managed-by: grafana-operator` label.
+enforceCacheLabels: "off"
+
 # -- Sets the `CLUSTER_DOMAIN` environment variable,
 # it defines how internal Kubernetes services managed by the operator are addressed.
 # By default, this is empty, and internal services are addressed without a cluster domain specified, i.e., a

main.go

@@ -27,29 +27,29 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/KimMachineGun/automemlimit/memlimit"
 	"go.uber.org/automaxprocs/maxprocs"
 	uberzap "go.uber.org/zap"
-	"go.uber.org/zap/zapcore"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/webhook"
-
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/labels"
-
-	corev1 "k8s.io/api/core/v1"
-	"sigs.k8s.io/controller-runtime/pkg/cache"
 
-	"github.com/KimMachineGun/automemlimit/memlimit"
 	"github.com/go-logr/logr"
+	"go.uber.org/zap/zapcore"
+
 	routev1 "github.com/openshift/api/route/v1"
+	v1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	discovery2 "k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
-	"github.com/grafana/grafana-operator/v5/controllers/model"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -61,6 +61,7 @@ import (
 	grafanav1beta1 "github.com/grafana/grafana-operator/v5/api/v1beta1"
 	"github.com/grafana/grafana-operator/v5/controllers"
 	"github.com/grafana/grafana-operator/v5/controllers/autodetect"
+	"github.com/grafana/grafana-operator/v5/controllers/model"
 	"github.com/grafana/grafana-operator/v5/embeds"
 	//+kubebuilder:scaffold:imports
 )
@@ -77,6 +78,8 @@ const (
 	// eg: 'partition in (customerA, customerB),environment!=qa'
 	// If empty of undefined, the operator will watch all CRs.
 	watchLabelSelectorsEnvVar = "WATCH_LABEL_SELECTORS"
+	// Opt in to enable new experimental cache limits by setting this to `safe` or `all`. Valid values are `off`, `safe` and `all`
+	enforceCacheLabelsEnvVar = "ENFORCE_CACHE_LABELS"
 	// clusterDomainEnvVar is the constant for env variable CLUSTER_DOMAIN, which specifies the cluster domain to use for addressing.
 	// By default, this is empty, and internal services are addressed without a cluster domain specified, i.e., a
 	// relative domain name that will resolve regardless of if a custom domain is configured for the cluster. If you
@@ -99,7 +102,7 @@ func init() {
 	//+kubebuilder:scaffold:scheme
 }
 
-func main() {
+func main() { // nolint:gocyclo
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
@@ -135,12 +138,25 @@ func main() {
 		setupLog.Error(err, "failed to adjust GOMAXPROCS")
 	}
 
+	// Detect environment variables
 	watchNamespace, _ := os.LookupEnv(watchNamespaceEnvVar)
 	watchNamespaceSelector, _ := os.LookupEnv(watchNamespaceEnvSelector)
 	watchLabelSelectors, _ := os.LookupEnv(watchLabelSelectorsEnvVar)
 	if watchLabelSelectors != "" {
 		setupLog.Info(fmt.Sprintf("sharding is enabled via %s=%s. Beware: Always label Grafana CRs before enabling to ensure labels are inherited. Existing Secrets/ConfigMaps referenced in CRs also need to be labeled to continue working.", watchLabelSelectorsEnvVar, watchLabelSelectors))
 	}
+
+	enforceCacheLabelsLevel, _ := os.LookupEnv(enforceCacheLabelsEnvVar)
+	enforceCacheLabels := false
+	switch enforceCacheLabelsLevel {
+	case "safe", "all":
+		enforceCacheLabels = true
+		setupLog.Info("label restrictions for cached resources are active", "level", enforceCacheLabelsLevel)
+	case "off", "":
+	default:
+		setupLog.Error(fmt.Errorf("invalid value %s for %s", enforceCacheLabelsLevel, enforceCacheLabelsEnvVar), "falling back to disabling cache enforcement")
+	}
+
 	clusterDomain, _ := os.LookupEnv(clusterDomainEnvVar)
 
 	// Fetch k8s api credentials and detect platform
@@ -157,13 +173,9 @@ func main() {
 	}
 
 	controllerOptions := ctrl.Options{
-		Scheme: scheme,
-		Metrics: metricsserver.Options{
-			BindAddress: metricsAddr,
-		},
-		WebhookServer: webhook.NewServer(webhook.Options{
-			Port: 9443,
-		}),
+		Scheme:                 scheme,
+		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
+		WebhookServer:          webhook.NewServer(webhook.Options{Port: 9443}),
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "f75f3bba.integreatly.org",
@@ -175,6 +187,39 @@ func main() {
 		setupLog.Error(err, fmt.Sprintf("unable to parse %s", watchLabelSelectorsEnvVar))
 		os.Exit(1) //nolint
 	}
+
+	if enforceCacheLabels {
+		var cacheLabelConfig cache.ByObject
+		if watchLabelSelectors != "" {
+			// When sharding, limit cache according to shard labels
+			cacheLabelConfig = cache.ByObject{Label: labelSelectors}
+			setupLog.Info(fmt.Sprintf("sharding is enabled via %s=%s. Beware: Always label Grafana CRs before enabling to ensure labels are inherited. Existing Secrets/ConfigMaps referenced in CRs also need to be labeled to continue working.", watchLabelSelectorsEnvVar, watchLabelSelectors))
+		} else {
+			// Otherwise limit it to managed-by label
+			cacheLabelConfig = cache.ByObject{Label: labels.SelectorFromSet(model.CommonLabels)}
+		}
+
+		// ConfigMaps and secrets stay fully cached until we implement support for bypassing the cache for referenced objects
+		controllerOptions.Cache.ByObject = map[client.Object]cache.ByObject{
+			&v1.Deployment{}:                cacheLabelConfig,
+			&corev1.Service{}:               cacheLabelConfig,
+			&corev1.ServiceAccount{}:        cacheLabelConfig,
+			&networkingv1.Ingress{}:         cacheLabelConfig,
+			&corev1.PersistentVolumeClaim{}: cacheLabelConfig,
+			&corev1.ConfigMap{}:             cacheLabelConfig, // Matching just labeled ConfigMaps and Secrets greatly reduces cache size
+			&corev1.Secret{}:                cacheLabelConfig, // Omitting labels or supporting custom labels would require changes in Grafana Reconciler
+		}
+		if isOpenShift {
+			controllerOptions.Cache.ByObject[&routev1.Route{}] = cacheLabelConfig
+		}
+		if enforceCacheLabelsLevel == "safe" {
+			controllerOptions.Client.Cache = &client.CacheOptions{
+				DisableFor: []client.Object{&corev1.ConfigMap{}, &corev1.Secret{}},
+			}
+		}
+	}
+
+	// Determine Operator scope
 	switch {
 	case strings.Contains(watchNamespace, ","):
 		// multi namespace scoped
@@ -185,7 +230,7 @@ func main() {
 		controllerOptions.Cache.DefaultNamespaces = getNamespaceConfig(watchNamespace, labelSelectors)
 		setupLog.Info("operator running in namespace scoped mode", "namespace", watchNamespace)
 	case strings.Contains(watchNamespaceSelector, ":"):
-		// namespace scoped
+		// multi namespace scoped
 		controllerOptions.Cache.DefaultNamespaces = getNamespaceConfigSelector(restConfig, watchNamespaceSelector, labelSelectors)
 		setupLog.Info("operator running in namespace scoped mode using namespace selector", "namespace", watchNamespace)
 
@@ -204,6 +249,7 @@ func main() {
 		os.Exit(1) //nolint
 	}
 
+	// Register controllers
 	if err = (&controllers.GrafanaReconciler{
 		Client:        mgr.GetClient(),
 		Scheme:        mgr.GetScheme(),