Telemetry optimization - Add K8sAPI and CAdvisorAPI response code telemetry in customMetrics (#1122)

* Add kubernetes response code telemetry

* Adding resource to the telemetry and resolving other comments

* Adding Cadvisor API status to the PR as well

* Separate maps for cadvisor and k8s

* Send only failure codes

---------

Co-authored-by: Janvi Jatakia (from Dev Box) <[email protected]>

Author: jatakiajanvi12

source/plugins/ruby/ApplicationInsightsUtility.rb

@@ -39,7 +39,6 @@ class ApplicationInsightsUtility
   end
 
   @@controllerType = {"daemonset" => "DS", "replicaset" => "RS"}
-
   def initialize
   end
 
@@ -323,5 +322,47 @@ def getWorkspaceCloud()
         $log.warn("Exception in AppInsightsUtility: getWorkspaceCloud - error: #{errorStr}")
       end
     end
+
+    def sendAPIResponseTelemetry(responseCode, resource, metricName, apiResponseCodeHash, apiResponseTelemetryTimeTracker)
+      begin
+        if (!responseCode.nil? && !responseCode.empty?)
+          if (!apiResponseCodeHash.has_key?(responseCode))
+            telemetryProps = {}
+            telemetryProps[resource] = 1
+            apiResponseCodeHash[responseCode] = telemetryProps
+          else
+            telemetryProps = apiResponseCodeHash[responseCode]
+            if (telemetryProps.nil?)
+              telemetryProps = {}
+            end
+            if (!telemetryProps.has_key?(resource))
+              telemetryProps[resource] = 1
+            else
+              telemetryProps[resource] += 1
+            end
+            apiResponseCodeHash[responseCode] = telemetryProps
+          end
+
+          timeDifference = (DateTime.now.to_time.to_i - apiResponseTelemetryTimeTracker).abs
+          timeDifferenceInMinutes = timeDifference / 60
+          if (timeDifferenceInMinutes >= Constants::TELEMETRY_FLUSH_INTERVAL_IN_MINUTES)
+            apiResponseTelemetryTimeTracker = DateTime.now.to_time.to_i
+            apiResponseCodeHash.each do |key, value|
+              value.each do |resourceName, count|
+                telemetryProps = {}
+                telemetryProps["Resource"] = resourceName
+                telemetryProps["ResponseCode"] = key
+                sendMetricTelemetry(metricName, count, telemetryProps)
+              end
+            end
+            apiResponseCodeHash.clear
+          end
+        end
+        return apiResponseTelemetryTimeTracker
+      rescue => err
+        $log.warn("Exception in AppInsightsUtility: sendAPIResponseTelemetry failed with an error: #{err}")
+      end
+    end
+
   end
 end

source/plugins/ruby/CAdvisorMetricsAPIClient.rb

@@ -39,6 +39,10 @@ class CAdvisorMetricsAPIClient
   @npmIntegrationAdvanced = ENV["TELEMETRY_NPM_INTEGRATION_METRICS_ADVANCED"]
   @subnetIpUsageMetrics = ENV["TELEMETRY_SUBNET_IP_USAGE_INTEGRATION_METRICS"]
 
+  @@CAdvisorApiResponseCodeHash = {}
+  @@CAdvisorApiResponseTelemetryTimeTracker = DateTime.now.to_time.to_i
+
+
   @os_type = ENV["OS_TYPE"]
   if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
     @LogPath = Constants::WINDOWS_LOG_PATH + "kubernetes_perf_log.txt"
@@ -963,6 +967,10 @@ def getResponse(winNode, relativeUri)
               @Log.info "Got response code #{response.code} from #{uri.request_uri}"
             end
           end
+          # Send telemetry for cAdvisor API response code if it is not success
+          if !response.nil? && !response.code.nil? && !response.code.start_with?("2")
+            @@CAdvisorApiResponseTelemetryTimeTracker = ApplicationInsightsUtility.sendAPIResponseTelemetry(response.code, relativeUri, "CAdvisorAPIStatus", @@CAdvisorApiResponseCodeHash, @@CAdvisorApiResponseTelemetryTimeTracker)
+          end
         end
       rescue => error
         @Log.warn("CAdvisor api request for #{cAdvisorUri} failed: #{error}")

source/plugins/ruby/KubernetesApiClient.rb

@@ -30,6 +30,9 @@ class KubernetesApiClient
   #@@IsValidRunningNode = nil
   #@@IsLinuxCluster = nil
   @@KubeSystemNamespace = "kube-system"
+  @@K8sApiResponseCodeHash = {}
+  @@K8sApiResponseTelemetryTimeTracker = DateTime.now.to_time.to_i
+
 
   @os_type = ENV["OS_TYPE"]
   if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
@@ -70,6 +73,11 @@ def getKubeResourceInfo(resource, api_group: nil)
               @Log.info "KubernetesAPIClient::getKubeResourceInfo : Making request to #{uri.request_uri} @ #{Time.now.utc.iso8601}"
               response = http.request(kubeApiRequest)
               @Log.info "KubernetesAPIClient::getKubeResourceInfo : Got response of #{response.code} for #{uri.request_uri} @ #{Time.now.utc.iso8601}"
+
+              # Send telemetry for response code if not success
+              if !response.nil? && !response.code.nil? && !response.code.start_with?("2")
+                @@K8sApiResponseTelemetryTimeTracker = ApplicationInsightsUtility.sendAPIResponseTelemetry(response.code, resource, "K8sAPIStatus", @@K8sApiResponseCodeHash, @@K8sApiResponseTelemetryTimeTracker)
+              end
             end
           end
         end
@@ -104,6 +112,11 @@ def getKubeResourceInfoV2(resource, api_group: nil)
               response = http.request(kubeApiRequest)
               responseCode = response.code
               @Log.info "KubernetesAPIClient::getKubeResourceInfoV2 : Got response of #{response.code} for #{uri.request_uri} @ #{Time.now.utc.iso8601}"
+              
+              # Send telemetry for response code if not success
+              if !response.nil? && !response.code.nil? && !response.code.start_with?("2")
+                @@K8sApiResponseTelemetryTimeTracker = ApplicationInsightsUtility.sendAPIResponseTelemetry(response.code, resource, "K8sAPIStatus", @@K8sApiResponseCodeHash, @@K8sApiResponseTelemetryTimeTracker)
+              end
             end
           end
         end