make multiline languages configurable (#1149)

* make multiline languages configurable

---------

Co-authored-by: Amol Agrawal <[email protected]>

build/common/installer/conf/azm-containers-parser-multiline.conf

@@ -1,7 +1,7 @@
 [MULTILINE_PARSER]
-    name          dotnet-multiline
+    name          dotnet
     type          regex
-    flush_timeout 1000 # milliseconds
+    flush_timeout 4000 # milliseconds, set to fluent-bit default https://github.com/fluent/fluent-bit/blob/master/include/fluent-bit/multiline/flb_ml.h#L50
 
     # Regex rules for multiline parsing
     # ---------------------------------

build/common/installer/scripts/fluent-bit-conf-customizer.rb

@@ -17,9 +17,12 @@ def is_number?(value)
   true if Integer(value) rescue false
 end
 
-def substituteMultiline(multilineLogging, new_contents)
+def substituteMultiline(multilineLogging, stacktraceLanguages, new_contents)
     if !multilineLogging.nil? && multilineLogging.to_s.downcase == "true"
-      new_contents = new_contents.gsub("#${MultilineEnabled}", "")
+      if !stacktraceLanguages.nil? && !stacktraceLanguages.empty?
+        new_contents = new_contents.gsub("#${MultilineEnabled}", "")
+        new_contents = new_contents.gsub("#${MultilineLanguages}", stacktraceLanguages)
+      end
       new_contents = new_contents.gsub("azm-containers-parser.conf", "azm-containers-parser-multiline.conf")
       # replace parser with multiline version. ensure running script multiple times does not have negative impact
       if (/[^\.]Parser\s{1,}docker/).match(new_contents)
@@ -43,6 +46,7 @@ def substituteFluentBitPlaceHolders
     memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"]
     ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"]
     multilineLogging = ENV["AZMON_MULTILINE_ENABLED"]
+    stacktraceLanguages = ENV["AZMON_MULTILINE_LANGUAGES"]
 
     serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval
     serviceIntervalSetting = "Flush         " + serviceInterval
@@ -79,13 +83,13 @@ def substituteFluentBitPlaceHolders
       new_contents = new_contents.gsub("\n    ${TAIL_IGNORE_OLDER}\n", "\n")
     end
 
-    new_contents = substituteMultiline(multilineLogging, new_contents)
+    new_contents = substituteMultiline(multilineLogging, stacktraceLanguages, new_contents)
     File.open(@fluent_bit_conf_path, "w") { |file| file.puts new_contents }
     puts "config::Successfully substituted the placeholders in fluent-bit.conf file"
 
     puts "config::Starting to substitute the placeholders in fluent-bit-common.conf file for log collection"
     text = File.read(@fluent_bit_common_conf_path)
-    new_contents = substituteMultiline(multilineLogging, text)
+    new_contents = substituteMultiline(multilineLogging, stacktraceLanguages, text)
     File.open(@fluent_bit_common_conf_path, "w") { |file| file.puts new_contents }
     puts "config::Successfully substituted the placeholders in fluent-bit-common.conf file"
 

build/common/installer/scripts/fluent-bit-geneva-conf-customizer.rb

@@ -38,6 +38,7 @@ def substituteFluentBitPlaceHolders(configFilePath)
     memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"]
     ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"]
     multilineLogging = ENV["AZMON_MULTILINE_ENABLED"]
+    stacktraceLanguages = ENV["AZMON_MULTILINE_LANGUAGES"]
     enableFluentBitThreading = ENV["ENABLE_FBIT_THREADING"]
 
     serviceInterval = is_valid_number?(interval) ? interval : @default_service_interval
@@ -82,7 +83,10 @@ def substituteFluentBitPlaceHolders(configFilePath)
     end
 
     if !multilineLogging.nil? && multilineLogging.to_s.downcase == "true"
-      new_contents = new_contents.gsub("#${MultilineEnabled}", "")
+      if !stacktraceLanguages.nil? && !stacktraceLanguages.empty?
+        new_contents = new_contents.gsub("#${MultilineEnabled}", "")
+        new_contents = new_contents.gsub("#${MultilineLanguages}", stacktraceLanguages)
+      end
       new_contents = new_contents.gsub("azm-containers-parser.conf", "azm-containers-parser-multiline.conf")
       # replace parser with multiline version. ensure running script multiple times does not have negative impact
       if (/[^\.]Parser\s{1,}docker/).match(text)

build/common/installer/scripts/tomlparser.rb

@@ -24,6 +24,7 @@
 @containerLogsRoute = "v2" # default for linux
 @adxDatabaseName = "containerinsights" # default for all configurations
 @logEnableMultiline = "false"
+@stacktraceLanguages = "go,java,python,dotnet"
 if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
   @containerLogsRoute = "v1" # default is v1 for windows until windows agent integrates windows ama
   # This path format is necessary for fluent-bit in windows
@@ -161,6 +162,27 @@ def populateSettingValuesFromConfigMap(parsedConfig)
           puts "config:: WARN: container logs V2 is disabled and is required for multiline logging. Disabling multiline logging"
           @logEnableMultiline = "false"
         end
+
+        multilineLanguages = parsedConfig[:log_collection_settings][:enable_multiline_logs][:stacktrace_languages]
+        if !multilineLanguages.nil?
+          if multilineLanguages.kind_of?(Array)
+            # Checking only for the first element to be string because toml enforces the arrays to contain elements of same type
+            # update stacktraceLanguages only if customer explicity overrode via configmap
+            #Empty the array to use the values from configmap
+            @stacktraceLanguages.clear
+            if multilineLanguages.length > 0 && multilineLanguages[0].kind_of?(String)
+              invalid_lang = multilineLanguages.any? { |lang| !["java", "python", "go", "dotnet"].include?(lang.downcase) }
+              if invalid_lang
+                puts "config::WARN: stacktrace languages contains invalid languages. Disabling multiline stacktrace logging"
+              else
+                @stacktraceLanguages = multilineLanguages.join(",").downcase
+                puts "config::Using config map setting for multiline languages"
+              end
+            else
+              puts "config::WARN: stacktrace languages is not an array of strings. Disabling multiline stacktrace logging"
+            end
+          end
+        end
       end
     rescue => errorStr
       ConfigParseErrorLogger.logError("Exception while reading config map settings for enabling multiline logs - #{errorStr}, using defaults, please check config map for errors")
@@ -252,6 +274,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   file.write("export AZMON_CONTAINER_LOG_SCHEMA_VERSION=#{@containerLogSchemaVersion}\n")
   file.write("export AZMON_ADX_DATABASE_NAME=#{@adxDatabaseName}\n")
   file.write("export AZMON_MULTILINE_ENABLED=#{@logEnableMultiline}\n")
+  file.write("export AZMON_MULTILINE_LANGUAGES=#{@stacktraceLanguages}\n")
   # Close file after writing all environment variables
   file.close
   puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' "
@@ -316,6 +339,8 @@ def get_command_windows(env_variable_name, env_variable_value)
     file.write(commands)
     commands = get_command_windows("AZMON_MULTILINE_ENABLED", @logEnableMultiline)
     file.write(commands)
+    commands = get_command_windows("AZMON_MULTILINE_LANGUAGES", @stacktraceLanguages)
+    file.write(commands)
     # Close file after writing all environment variables
     file.close
     puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' "

build/linux/installer/conf/fluent-bit-geneva.conf

@@ -33,4 +33,4 @@
 #${MultilineEnabled}    Name multiline
 #${MultilineEnabled}    Match geneva.container.log.*
 #${MultilineEnabled}    multiline.key_content log
-#${MultilineEnabled}    multiline.parser go, java, python, dotnet-multiline
+#${MultilineEnabled}    multiline.parser #${MultilineLanguages}

build/linux/installer/conf/fluent-bit.conf

@@ -43,4 +43,4 @@
 #${MultilineEnabled}    Name multiline
 #${MultilineEnabled}    Match oms.container.log.la.*
 #${MultilineEnabled}    multiline.key_content log
-#${MultilineEnabled}    multiline.parser go, java, python, dotnet-multiline
+#${MultilineEnabled}    multiline.parser #${MultilineLanguages}

build/windows/installer/conf/fluent-bit-geneva.conf

@@ -31,4 +31,4 @@
 #${MultilineEnabled}    Name multiline
 #${MultilineEnabled}    Match geneva.container.log.*
 #${MultilineEnabled}    multiline.key_content log
-#${MultilineEnabled}    multiline.parser go, java, python, dotnet-multiline
+#${MultilineEnabled}    multiline.parser #${MultilineLanguages}

build/windows/installer/conf/fluent-bit.conf

@@ -42,4 +42,4 @@
 #${MultilineEnabled}    Name multiline
 #${MultilineEnabled}    Match oms.container.log.la.*
 #${MultilineEnabled}    multiline.key_content log
-#${MultilineEnabled}    multiline.parser go, java, python, dotnet-multiline
+#${MultilineEnabled}    multiline.parser #${MultilineLanguages}

kubernetes/container-azm-ms-agentconfig.yaml

@@ -48,9 +48,11 @@ data:
           # See documentation at https://aka.ms/ContainerLogv2 for benefits of v2 schema over v1 schema before opting for "v2" schema
           # containerlog_schema_version = "v2"
        #[log_collection_settings.enable_multiline_logs]
-          # fluent-bit based multiline log collection for .NET, Go, Java, and Python stacktraces.
-          # if enabled will also stitch together container logs split by docker/cri due to size limits(16KB per log line)
+          # fluent-bit based multiline log collection for .NET, Go, Java, and Python stacktraces. Update stacktrace_languages to specificy which languages to collect stacktraces for(valid inputs: go, java, python, dotnet). Default enables all.
+          # NOTE: for better performance consider enabling only for languages that are needed. Dotnet is experimental and may not work in all cases.
+          # If enabled will also stitch together container logs split by docker/cri due to size limits(16KB per log line)
           # enabled = "false"
+          # stacktrace_languages = ["go", "java", "python", "dotnet"] 
 
 
   prometheus-data-collection-settings: |-

source/plugins/ruby/CAdvisorMetricsAPIClient.rb

@@ -27,6 +27,7 @@ class CAdvisorMetricsAPIClient
   @clusterContainerLogEnrich = ENV["AZMON_CLUSTER_CONTAINER_LOG_ENRICH"]
   @clusterContainerLogSchemaVersion = ENV["AZMON_CONTAINER_LOG_SCHEMA_VERSION"]
   @clusterMultilineEnabled = ENV["AZMON_MULTILINE_ENABLED"]
+  @clusterMultilineLanguages = ENV["AZMON_MULTILINE_LANGUAGES"]
 
   @dsPromInterval = ENV["TELEMETRY_DS_PROM_INTERVAL"]
   @dsPromFieldPassCount = ENV["TELEMETRY_DS_PROM_FIELDPASS_LENGTH"]
@@ -299,6 +300,9 @@ def getContainerCpuMetricItems(metricJSON, hostName, cpuMetricNameToCollect, met
                     end
                     if (!@clusterMultilineEnabled.nil? && !@clusterMultilineEnabled.empty?)
                       telemetryProps["multilineEnabled"] = @clusterMultilineEnabled
+                      if (!@clusterMultilineLanguages.nil? && !@clusterMultilineLanguages.empty?)
+                        telemetryProps["multilineLanguages"] = @clusterMultilineLanguages
+                      end
                     end
                     ApplicationInsightsUtility.sendMetricTelemetry(metricNametoReturn, metricValue, telemetryProps)
                   end