diff --git a/build/common/installer/conf/azm-containers-parser-multiline.conf b/build/common/installer/conf/azm-containers-parser-multiline.conf index dec075e68..56666fa33 100644 --- a/build/common/installer/conf/azm-containers-parser-multiline.conf +++ b/build/common/installer/conf/azm-containers-parser-multiline.conf @@ -1,7 +1,7 @@ [MULTILINE_PARSER] - name dotnet-multiline + name dotnet type regex - flush_timeout 1000 # milliseconds + flush_timeout 4000 # milliseconds, set to fluent-bit default https://github.com/fluent/fluent-bit/blob/master/include/fluent-bit/multiline/flb_ml.h#L50 # Regex rules for multiline parsing # --------------------------------- diff --git a/build/common/installer/scripts/fluent-bit-conf-customizer.rb b/build/common/installer/scripts/fluent-bit-conf-customizer.rb index 05a27348d..1d5ad7ca6 100644 --- a/build/common/installer/scripts/fluent-bit-conf-customizer.rb +++ b/build/common/installer/scripts/fluent-bit-conf-customizer.rb @@ -17,9 +17,12 @@ def is_number?(value) true if Integer(value) rescue false end -def substituteMultiline(multilineLogging, new_contents) +def substituteMultiline(multilineLogging, stacktraceLanguages, new_contents) if !multilineLogging.nil? && multilineLogging.to_s.downcase == "true" - new_contents = new_contents.gsub("#${MultilineEnabled}", "") + if !stacktraceLanguages.nil? && !stacktraceLanguages.empty? + new_contents = new_contents.gsub("#${MultilineEnabled}", "") + new_contents = new_contents.gsub("#${MultilineLanguages}", stacktraceLanguages) + end new_contents = new_contents.gsub("azm-containers-parser.conf", "azm-containers-parser-multiline.conf") # replace parser with multiline version. ensure running script multiple times does not have negative impact if (/[^\.]Parser\s{1,}docker/).match(new_contents) @@ -43,6 +46,7 @@ def substituteFluentBitPlaceHolders memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"] ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"] multilineLogging = ENV["AZMON_MULTILINE_ENABLED"] + stacktraceLanguages = ENV["AZMON_MULTILINE_LANGUAGES"] serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval serviceIntervalSetting = "Flush " + serviceInterval @@ -79,13 +83,13 @@ def substituteFluentBitPlaceHolders new_contents = new_contents.gsub("\n ${TAIL_IGNORE_OLDER}\n", "\n") end - new_contents = substituteMultiline(multilineLogging, new_contents) + new_contents = substituteMultiline(multilineLogging, stacktraceLanguages, new_contents) File.open(@fluent_bit_conf_path, "w") { |file| file.puts new_contents } puts "config::Successfully substituted the placeholders in fluent-bit.conf file" puts "config::Starting to substitute the placeholders in fluent-bit-common.conf file for log collection" text = File.read(@fluent_bit_common_conf_path) - new_contents = substituteMultiline(multilineLogging, text) + new_contents = substituteMultiline(multilineLogging, stacktraceLanguages, text) File.open(@fluent_bit_common_conf_path, "w") { |file| file.puts new_contents } puts "config::Successfully substituted the placeholders in fluent-bit-common.conf file" diff --git a/build/common/installer/scripts/fluent-bit-geneva-conf-customizer.rb b/build/common/installer/scripts/fluent-bit-geneva-conf-customizer.rb index 7ed43b160..5604fa039 100644 --- a/build/common/installer/scripts/fluent-bit-geneva-conf-customizer.rb +++ b/build/common/installer/scripts/fluent-bit-geneva-conf-customizer.rb @@ -38,6 +38,7 @@ def substituteFluentBitPlaceHolders(configFilePath) memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"] ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"] multilineLogging = ENV["AZMON_MULTILINE_ENABLED"] + stacktraceLanguages = ENV["AZMON_MULTILINE_LANGUAGES"] enableFluentBitThreading = ENV["ENABLE_FBIT_THREADING"] serviceInterval = is_valid_number?(interval) ? interval : @default_service_interval @@ -82,7 +83,10 @@ def substituteFluentBitPlaceHolders(configFilePath) end if !multilineLogging.nil? && multilineLogging.to_s.downcase == "true" - new_contents = new_contents.gsub("#${MultilineEnabled}", "") + if !stacktraceLanguages.nil? && !stacktraceLanguages.empty? + new_contents = new_contents.gsub("#${MultilineEnabled}", "") + new_contents = new_contents.gsub("#${MultilineLanguages}", stacktraceLanguages) + end new_contents = new_contents.gsub("azm-containers-parser.conf", "azm-containers-parser-multiline.conf") # replace parser with multiline version. ensure running script multiple times does not have negative impact if (/[^\.]Parser\s{1,}docker/).match(text) diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index 651d715ec..e80ca2b96 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -24,6 +24,7 @@ @containerLogsRoute = "v2" # default for linux @adxDatabaseName = "containerinsights" # default for all configurations @logEnableMultiline = "false" +@stacktraceLanguages = "go,java,python,dotnet" if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 @containerLogsRoute = "v1" # default is v1 for windows until windows agent integrates windows ama # This path format is necessary for fluent-bit in windows @@ -161,6 +162,27 @@ def populateSettingValuesFromConfigMap(parsedConfig) puts "config:: WARN: container logs V2 is disabled and is required for multiline logging. Disabling multiline logging" @logEnableMultiline = "false" end + + multilineLanguages = parsedConfig[:log_collection_settings][:enable_multiline_logs][:stacktrace_languages] + if !multilineLanguages.nil? + if multilineLanguages.kind_of?(Array) + # Checking only for the first element to be string because toml enforces the arrays to contain elements of same type + # update stacktraceLanguages only if customer explicity overrode via configmap + #Empty the array to use the values from configmap + @stacktraceLanguages.clear + if multilineLanguages.length > 0 && multilineLanguages[0].kind_of?(String) + invalid_lang = multilineLanguages.any? { |lang| !["java", "python", "go", "dotnet"].include?(lang.downcase) } + if invalid_lang + puts "config::WARN: stacktrace languages contains invalid languages. Disabling multiline stacktrace logging" + else + @stacktraceLanguages = multilineLanguages.join(",").downcase + puts "config::Using config map setting for multiline languages" + end + else + puts "config::WARN: stacktrace languages is not an array of strings. Disabling multiline stacktrace logging" + end + end + end end rescue => errorStr ConfigParseErrorLogger.logError("Exception while reading config map settings for enabling multiline logs - #{errorStr}, using defaults, please check config map for errors") @@ -252,6 +274,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) file.write("export AZMON_CONTAINER_LOG_SCHEMA_VERSION=#{@containerLogSchemaVersion}\n") file.write("export AZMON_ADX_DATABASE_NAME=#{@adxDatabaseName}\n") file.write("export AZMON_MULTILINE_ENABLED=#{@logEnableMultiline}\n") + file.write("export AZMON_MULTILINE_LANGUAGES=#{@stacktraceLanguages}\n") # Close file after writing all environment variables file.close puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' " @@ -316,6 +339,8 @@ def get_command_windows(env_variable_name, env_variable_value) file.write(commands) commands = get_command_windows("AZMON_MULTILINE_ENABLED", @logEnableMultiline) file.write(commands) + commands = get_command_windows("AZMON_MULTILINE_LANGUAGES", @stacktraceLanguages) + file.write(commands) # Close file after writing all environment variables file.close puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' " diff --git a/build/linux/installer/conf/fluent-bit-geneva.conf b/build/linux/installer/conf/fluent-bit-geneva.conf index 1aeacdba1..9376b434f 100644 --- a/build/linux/installer/conf/fluent-bit-geneva.conf +++ b/build/linux/installer/conf/fluent-bit-geneva.conf @@ -33,4 +33,4 @@ #${MultilineEnabled} Name multiline #${MultilineEnabled} Match geneva.container.log.* #${MultilineEnabled} multiline.key_content log -#${MultilineEnabled} multiline.parser go, java, python, dotnet-multiline +#${MultilineEnabled} multiline.parser #${MultilineLanguages} diff --git a/build/linux/installer/conf/fluent-bit.conf b/build/linux/installer/conf/fluent-bit.conf index 7ed4990b0..a9a780448 100644 --- a/build/linux/installer/conf/fluent-bit.conf +++ b/build/linux/installer/conf/fluent-bit.conf @@ -43,4 +43,4 @@ #${MultilineEnabled} Name multiline #${MultilineEnabled} Match oms.container.log.la.* #${MultilineEnabled} multiline.key_content log -#${MultilineEnabled} multiline.parser go, java, python, dotnet-multiline +#${MultilineEnabled} multiline.parser #${MultilineLanguages} diff --git a/build/windows/installer/conf/fluent-bit-geneva.conf b/build/windows/installer/conf/fluent-bit-geneva.conf index 7bfc497fd..44f69232c 100644 --- a/build/windows/installer/conf/fluent-bit-geneva.conf +++ b/build/windows/installer/conf/fluent-bit-geneva.conf @@ -31,4 +31,4 @@ #${MultilineEnabled} Name multiline #${MultilineEnabled} Match geneva.container.log.* #${MultilineEnabled} multiline.key_content log -#${MultilineEnabled} multiline.parser go, java, python, dotnet-multiline +#${MultilineEnabled} multiline.parser #${MultilineLanguages} diff --git a/build/windows/installer/conf/fluent-bit.conf b/build/windows/installer/conf/fluent-bit.conf index 1404b83cd..1a2d2e38b 100644 --- a/build/windows/installer/conf/fluent-bit.conf +++ b/build/windows/installer/conf/fluent-bit.conf @@ -42,4 +42,4 @@ #${MultilineEnabled} Name multiline #${MultilineEnabled} Match oms.container.log.la.* #${MultilineEnabled} multiline.key_content log -#${MultilineEnabled} multiline.parser go, java, python, dotnet-multiline +#${MultilineEnabled} multiline.parser #${MultilineLanguages} diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index a1dc1e0cd..83ea366d0 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -48,9 +48,11 @@ data: # See documentation at https://aka.ms/ContainerLogv2 for benefits of v2 schema over v1 schema before opting for "v2" schema # containerlog_schema_version = "v2" #[log_collection_settings.enable_multiline_logs] - # fluent-bit based multiline log collection for .NET, Go, Java, and Python stacktraces. - # if enabled will also stitch together container logs split by docker/cri due to size limits(16KB per log line) + # fluent-bit based multiline log collection for .NET, Go, Java, and Python stacktraces. Update stacktrace_languages to specificy which languages to collect stacktraces for(valid inputs: go, java, python, dotnet). Default enables all. + # NOTE: for better performance consider enabling only for languages that are needed. Dotnet is experimental and may not work in all cases. + # If enabled will also stitch together container logs split by docker/cri due to size limits(16KB per log line) # enabled = "false" + # stacktrace_languages = ["go", "java", "python", "dotnet"] prometheus-data-collection-settings: |- diff --git a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb index c91be5746..487b6227d 100644 --- a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb +++ b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb @@ -27,6 +27,7 @@ class CAdvisorMetricsAPIClient @clusterContainerLogEnrich = ENV["AZMON_CLUSTER_CONTAINER_LOG_ENRICH"] @clusterContainerLogSchemaVersion = ENV["AZMON_CONTAINER_LOG_SCHEMA_VERSION"] @clusterMultilineEnabled = ENV["AZMON_MULTILINE_ENABLED"] + @clusterMultilineLanguages = ENV["AZMON_MULTILINE_LANGUAGES"] @dsPromInterval = ENV["TELEMETRY_DS_PROM_INTERVAL"] @dsPromFieldPassCount = ENV["TELEMETRY_DS_PROM_FIELDPASS_LENGTH"] @@ -299,6 +300,9 @@ def getContainerCpuMetricItems(metricJSON, hostName, cpuMetricNameToCollect, met end if (!@clusterMultilineEnabled.nil? && !@clusterMultilineEnabled.empty?) telemetryProps["multilineEnabled"] = @clusterMultilineEnabled + if (!@clusterMultilineLanguages.nil? && !@clusterMultilineLanguages.empty?) + telemetryProps["multilineLanguages"] = @clusterMultilineLanguages + end end ApplicationInsightsUtility.sendMetricTelemetry(metricNametoReturn, metricValue, telemetryProps) end