add integration for azure subnet ip usage (#774)

* add integration for azure cni subnet ip usage

* exclude unfixed cve & remove fixed one

Author: vishiy

.trivyignore

@@ -16,4 +16,4 @@ CVE-2021-31799
 CVE-2021-28965
 
 #dpkg vulnerability in ubuntu
-CVE-2022-1664
+CVE-2022-1304

build/linux/installer/conf/telegraf.conf

@@ -805,4 +805,23 @@
 #     ClusterName = "$TELEMETRY_CLUSTER_NAME"
 #     ClusterType = "$TELEMETRY_CLUSTER_TYPE"
 #     Computer = "placeholder_hostname"
-#     ControllerType = "$CONTROLLER_TYPE"
+#     ControllerType = "$CONTROLLER_TYPE"
+
+## ip subnet usage
+[[inputs.prometheus]]
+  #name_prefix="container.azm.ms/"
+  ## An array of urls to scrape metrics from.
+  urls = $AZMON_INTEGRATION_SUBNET_IP_USAGE_METRICS_URL_LIST_NODE
+
+  metric_version = 2
+  url_tag = "scrapeUrl"
+
+  ## Use bearer token for authorization. ('bearer_token' takes priority)
+  bearer_token = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+
+  ## Specify timeout duration for slower prometheus clients (default is 3s)
+  response_timeout = "15s"
+
+  ## Optional TLS Config
+  tls_ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  insecure_skip_verify = true

build/linux/installer/scripts/tomlparser-npm-config.rb

@@ -18,6 +18,9 @@
 @npm_node_urls = "[\"http://$NODE_IP:10091/node-metrics\"]"
 @npm_cluster_urls="[\"http://npm-metrics-cluster-service.kube-system:9000/cluster-metrics\"]"
 @npm_basic_drop_metrics_cluster = "[\"npm_ipset_counts\"]"
+@collect_subnet_ip_usage_metrics = false
+@azure_subnet_ip_usage_metrics_node_urls = "[\"http://$NODE_IP:10092/metrics\"]"
+@azure_subnet_ip_usage_default_setting = "[]"
 @tgfConfigFileDS = "/etc/opt/microsoft/docker-cimprov/telegraf.conf"
 @tgfConfigFileRS = "/etc/opt/microsoft/docker-cimprov/telegraf-rs.conf"
 @replicaset = "replicaset"
@@ -43,6 +46,21 @@ def parseConfigMap
 
 # Use the ruby structure created after config parsing to set the right values to be used as environment variables
 def populateSettingValuesFromConfigMap(parsedConfig)
+  begin
+    if !parsedConfig.nil? && !parsedConfig[:integrations].nil? && !parsedConfig[:integrations][:azure_subnet_ip_usage].nil? && !parsedConfig[:integrations][:azure_subnet_ip_usage][:enabled].nil?
+        azure_subnet_ip_usage_metrics = parsedConfig[:integrations][:azure_subnet_ip_usage][:enabled].to_s
+        puts "config::azure_subnet_ip_usage::got:integrations.azure_subnet_ip_usage.enabled='#{azure_subnet_ip_usage_metrics}'"
+        if !azure_subnet_ip_usage_metrics.nil? && azure_subnet_ip_usage_metrics.strip.casecmp("true") == 0
+            @collect_azure_subnet_ip_usage_metrics = true
+        else
+            @collect_azure_subnet_ip_usage_metrics = false
+        end
+        puts "config::azure_subnet_ip_usage::got:integrations.azure_subnet_ip_usage.enabled=#{@collect_azure_subnet_ip_usage_metrics}"
+    end
+  rescue => errorStr
+    puts "config::npm::error:Exception while reading config settings for azure_subnet_ip_usage setting - #{errorStr}, using defaults"
+    @collect_azure_subnet_ip_usage_metrics = false
+  end
   begin
     if !parsedConfig.nil? && !parsedConfig[:integrations].nil? && !parsedConfig[:integrations][:azure_network_policy_manager].nil? && !parsedConfig[:integrations][:azure_network_policy_manager][:collect_advanced_metrics].nil?
         advanced_npm_metrics = parsedConfig[:integrations][:azure_network_policy_manager][:collect_advanced_metrics].to_s
@@ -76,18 +94,19 @@ def populateSettingValuesFromConfigMap(parsedConfig)
 end
 
 @configSchemaVersion = ENV["AZMON_AGENT_CFG_SCHEMA_VERSION"]
-puts "****************Start NPM Config Processing********************"
+puts "****************Start NPM & subnet ip usage integrations Config Processing********************"
 if !@configSchemaVersion.nil? && !@configSchemaVersion.empty? && @configSchemaVersion.strip.casecmp("v1") == 0 #note v1 is the only supported schema version , so hardcoding it
   configMapSettings = parseConfigMap
   if !configMapSettings.nil?
     populateSettingValuesFromConfigMap(configMapSettings)
   end
 else
   if (File.file?(@configMapMountPath))
-    ConfigParseErrorLogger.logError("config::npm::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults, please use supported schema version")
+    ConfigParseErrorLogger.logError("config::integrations::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults, please use supported schema version")
   end
   @collect_basic_npm_metrics = false
   @collect_advanced_npm_metrics = false
+  @collect_azure_subnet_ip_usage_metrics = false
 end
 
 
@@ -99,7 +118,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   tgfConfigFile = @tgfConfigFileRS
 end
 
-#replace place holders in configuration file
+#replace place holders in configuration file for npm integration
 tgfConfig = File.read(tgfConfigFile) #read returns only after closing the file
 
 if @collect_advanced_npm_metrics == true
@@ -116,8 +135,19 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   tgfConfig = tgfConfig.gsub("$AZMON_INTEGRATION_NPM_METRICS_DROP_LIST_CLUSTER", @npm_default_setting)
 end
 
+#replace place holders in configuration file for subnet ip usage integration
+if @collect_azure_subnet_ip_usage_metrics == true
+  tgfConfig = tgfConfig.gsub("$AZMON_INTEGRATION_SUBNET_IP_USAGE_METRICS_URL_LIST_NODE", @azure_subnet_ip_usage_metrics_node_urls)
+else
+  tgfConfig = tgfConfig.gsub("$AZMON_INTEGRATION_SUBNET_IP_USAGE_METRICS_URL_LIST_NODE", @azure_subnet_ip_usage_default_setting)
+end
+
+File.open(tgfConfigFile, "w") { |file| file.puts tgfConfig } # 'file' will be closed here after it goes out of scope
+puts "config::integrations::Successfully substituted the placeholders for integrations into #{tgfConfigFile} file for #{controller}"
+
+
 File.open(tgfConfigFile, "w") { |file| file.puts tgfConfig } # 'file' will be closed here after it goes out of scope
-puts "config::npm::Successfully substituted the NPM placeholders into #{tgfConfigFile} file for #{controller}"
+puts "config::integrations::Successfully substituted the integrations placeholders into #{tgfConfigFile} file for #{controller}"
 
 # Write the telemetry to file, so that they can be set as environment variables
 telemetryFile = File.open("integration_npm_config_env_var", "w")
@@ -128,9 +158,12 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   elsif @collect_basic_npm_metrics == true
     telemetryFile.write("export TELEMETRY_NPM_INTEGRATION_METRICS_BASIC=1\n")
   end
+  if @collect_azure_subnet_ip_usage_metrics == true
+    telemetryFile.write("export TELEMETRY_SUBNET_IP_USAGE_INTEGRATION_METRICS=1\n")
+  end
   # Close file after writing all environment variables
   telemetryFile.close
 else
-  puts "config::npm::Exception while opening file for writing NPM telemetry environment variables"
+  puts "config::integrations::Exception while opening file for writing Integrations telemetry environment variables"
   puts "****************End NPM Config Processing********************"
 end

kubernetes/container-azm-ms-agentconfig.yaml

@@ -141,6 +141,8 @@ data:
     [integrations.azure_network_policy_manager]
         collect_basic_metrics = false
         collect_advanced_metrics = false
+    [integrations.azure_subnet_ip_usage]
+        enabled = false
 
 # Doc - https://github.com/microsoft/Docker-Provider/blob/ci_prod/Documentation/AgentSettings/ReadMe.md
   agent-settings: |-

source/plugins/ruby/CAdvisorMetricsAPIClient.rb

@@ -36,6 +36,7 @@ class CAdvisorMetricsAPIClient
   @containerLogsRoute = ENV["AZMON_CONTAINER_LOGS_ROUTE"]
   @npmIntegrationBasic = ENV["TELEMETRY_NPM_INTEGRATION_METRICS_BASIC"]
   @npmIntegrationAdvanced = ENV["TELEMETRY_NPM_INTEGRATION_METRICS_ADVANCED"]
+  @subnetIpUsageMetrics = ENV["TELEMETRY_SUBNET_IP_USAGE_INTEGRATION_METRICS"]
 
   @os_type = ENV["OS_TYPE"]
   if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
@@ -282,6 +283,10 @@ def getContainerCpuMetricItems(metricJSON, hostName, cpuMetricNameToCollect, met
                     elsif (!@npmIntegrationBasic.nil? && !@npmIntegrationBasic.empty?)
                       telemetryProps["int-npm-b"] = "1"
                     end
+                    # telemetry for subnet ip usage integration
+                    if (!@subnetIpUsageMetrics.nil? && !@subnetIpUsageMetrics.empty?)
+                      telemetryProps["int-ipsubnetusage"] = "1"
+                    end
                     #telemetry for Container log schema version clusterContainerLogSchemaVersion
                     if (!@clusterContainerLogSchemaVersion.nil? && !@clusterContainerLogSchemaVersion.empty?)
                       telemetryProps["containerLogVer"] = @clusterContainerLogSchemaVersion