add feature-gate to allow cri metrics instead of cAdvisor

Author: bravebeaver

pkg/agent/baker.go

@@ -327,6 +327,12 @@ func ValidateAndSetLinuxNodeBootstrappingConfiguration(config *datamodel.NodeBoo
 		!IsKubernetesVersionGe(config.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, "1.25.0") {
 		kubeletFlags["--feature-gates"] = addFeatureGateString(kubeletFlags["--feature-gates"], "DisableAcceleratorUsageMetrics", false)
 	}
+
+	// https://kubernetes.io/docs/reference/instrumentation/cri-pod-container-metrics/
+	// use cri metrics instead of cAdvisor
+	if IsKubeletCriMetricsEnabled(config) {
+		kubeletFlags["--feature-gates"] = addFeatureGateString(kubeletFlags["--feature-gates"], "PodAndContainerStatsFromCRI", true)
+	}
 }
 
 func validateAndSetWindowsNodeBootstrappingConfiguration(config *datamodel.NodeBootstrappingConfiguration) {

pkg/agent/utils.go

@@ -444,6 +444,13 @@ func IsKubeletServingCertificateRotationEnabled(config *datamodel.NodeBootstrapp
 	return config.KubeletConfig["--rotate-server-certificates"] == "true"
 }
 
+func IsKubeletCriMetricsEnabled(config *datamodel.NodeBootstrappingConfiguration) bool {
+	if config == nil || config.KubeletConfig == nil {
+		return false
+	}
+	return config.KubeletConfig["--pod-container-stats-cri"] == "true"
+}
+
 func getAKSKubeletConfiguration(kc map[string]string) *datamodel.AKSKubeletConfiguration {
 	kubeletConfig := &datamodel.AKSKubeletConfiguration{
 		APIVersion:    "kubelet.config.k8s.io/v1beta1",

pkg/agent/utils_test.go

@@ -69,6 +69,41 @@ func TestGetKubeletConfigFileFromFlags(t *testing.T) {
 	}
 }
 
+func getTelescopeKubeletConfig() map[string]string {
+	kc := map[string]string{
+		"--address":                           "0.0.0.0",
+		"--pod-manifest-path":                 "/etc/kubernetes/manifests",
+		"--cluster-domain":                    "cluster.local",
+		"--cluster-dns":                       "10.0.0.10",
+		"--cgroups-per-qos":                   "true",
+		"--tls-cert-file":                     "/etc/kubernetes/certs/kubeletserver.crt",
+		"--tls-private-key-file":              "/etc/kubernetes/certs/kubeletserver.key",
+		"--tls-cipher-suites":                 "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256", //nolint:lll
+		"--max-pods":                          "110",
+		"--node-status-update-frequency":      "10s",
+		"--node-status-report-frequency":      "5m0s",
+		"--image-gc-high-threshold":           "85",
+		"--image-gc-low-threshold":            "80",
+		"--event-qps":                         "0",
+		"--pod-max-pids":                      "-1",
+		"--enforce-node-allocatable":          "pods",
+		"--streaming-connection-idle-timeout": "4h0m0s",
+		"--rotate-certificates":               "true",
+		"--read-only-port":                    "10255",
+		"--protect-kernel-defaults":           "true",
+		"--resolv-conf":                       "/etc/resolv.conf",
+		"--anonymous-auth":                    "false",
+		"--client-ca-file":                    "/etc/kubernetes/certs/ca.crt",
+		"--authentication-token-webhook":      "true",
+		"--authorization-mode":                "Webhook",
+		"--eviction-hard":                     "memory.available<750Mi,nodefs.available<10%,nodefs.inodesFree<5%",
+		"--feature-gates":                     "PodAndContainerStatsFromCRI=true",
+		"--system-reserved":                   "cpu=2,memory=1Gi",
+		"--kube-reserved":                     "cpu=100m,memory=1638Mi",
+	}
+	return kc
+}
+
 func getExampleKcWithNodeStatusReportFrequency() map[string]string {
 	kc := map[string]string{
 		"--address":                           "0.0.0.0",
@@ -222,6 +257,74 @@ var expectedKubeletJSON = `{
     "seccompDefault": true
 }`
 
+var expectedKubeletTelescopeJSON = `{
+    "kind": "KubeletConfiguration",
+    "apiVersion": "kubelet.config.k8s.io/v1beta1",
+    "staticPodPath": "/etc/kubernetes/manifests",
+    "address": "0.0.0.0",
+    "readOnlyPort": 10255,
+    "tlsCertFile": "/etc/kubernetes/certs/kubeletserver.crt",
+    "tlsPrivateKeyFile": "/etc/kubernetes/certs/kubeletserver.key",
+    "tlsCipherSuites": [
+        "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+        "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+        "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
+        "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+        "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
+        "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+        "TLS_RSA_WITH_AES_256_GCM_SHA384",
+        "TLS_RSA_WITH_AES_128_GCM_SHA256"
+    ],
+    "rotateCertificates": true,
+    "authentication": {
+        "x509": {
+            "clientCAFile": "/etc/kubernetes/certs/ca.crt"
+        },
+        "webhook": {
+            "enabled": true
+        },
+        "anonymous": {}
+    },
+    "authorization": {
+        "mode": "Webhook",
+        "webhook": {}
+    },
+    "eventRecordQPS": 0,
+    "clusterDomain": "cluster.local",
+    "clusterDNS": [
+        "10.0.0.10"
+    ],
+    "streamingConnectionIdleTimeout": "4h0m0s",
+    "nodeStatusUpdateFrequency": "10s",
+    "nodeStatusReportFrequency": "5m0s",
+    "imageGCHighThresholdPercent": 85,
+    "imageGCLowThresholdPercent": 80,
+    "cgroupsPerQOS": true,
+    "maxPods": 110,
+    "podPidsLimit": -1,
+    "resolvConf": "/etc/resolv.conf",
+    "evictionHard": {
+        "memory.available": "750Mi",
+        "nodefs.available": "10%",
+        "nodefs.inodesFree": "5%"
+    },
+    "protectKernelDefaults": true,
+    "featureGates": {
+        "PodAndContainerStatsFromCRI": true
+    },
+    "systemReserved": {
+        "cpu": "2",
+        "memory": "1Gi"
+    },
+    "kubeReserved": {
+        "cpu": "100m",
+        "memory": "1638Mi"
+    },
+    "enforceNodeAllocatable": [
+        "pods"
+    ]
+}`
+
 var expectedKubeletJSONWithNodeStatusReportFrequency = `{
     "kind": "KubeletConfiguration",
     "apiVersion": "kubelet.config.k8s.io/v1beta1",
@@ -462,6 +565,16 @@ func TestGetKubeletConfigFileFlagsWithNodeStatusReportFrequency(t *testing.T) {
 	}
 }
 
+func TestGetKubeletConfigFileFlagsUsingCriMetrics(t *testing.T) {
+	kc := getTelescopeKubeletConfig()
+	customKc := &datamodel.CustomKubeletConfig{}
+	configFileStr := GetKubeletConfigFileContent(kc, customKc)
+	diff := cmp.Diff(expectedKubeletTelescopeJSON, configFileStr)
+	if diff != "" {
+		t.Errorf("Generated config file is different than expected: %s", diff)
+	}
+}
+
 func TestGetKubeletConfigFileFromFlagsWithContainerLogMaxSize(t *testing.T) {
 	kc := getExampleKcWithContainerLogMaxSize()
 	customKc := &datamodel.CustomKubeletConfig{