diff --git a/Documentation/MultiTenancyLogging/AzureMonitorContainers.json b/Documentation/MultiTenancyLogging/AdvancedMode/AzureMonitorContainers_AdvancedMode_Grafana.json similarity index 84% rename from Documentation/MultiTenancyLogging/AzureMonitorContainers.json rename to Documentation/MultiTenancyLogging/AdvancedMode/AzureMonitorContainers_AdvancedMode_Grafana.json index dbd482a40..fc906c369 100644 --- a/Documentation/MultiTenancyLogging/AzureMonitorContainers.json +++ b/Documentation/MultiTenancyLogging/AdvancedMode/AzureMonitorContainers_AdvancedMode_Grafana.json @@ -24,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 221, + "id": 259, "links": [], "liveNow": false, "panels": [ @@ -53,6 +53,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -66,6 +67,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -146,6 +148,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -159,6 +162,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -238,6 +242,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -251,6 +256,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -330,6 +336,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -343,6 +350,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -422,6 +430,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -435,6 +444,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -514,6 +524,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -527,6 +538,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -607,6 +619,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -620,6 +633,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -699,6 +713,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -712,6 +727,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -777,7 +793,7 @@ "refId": "A" } ], - "title": "Fluent-bit output process records/s by Node", + "title": "Fluent-bit output forwarded records/s by Node", "type": "timeseries" }, { @@ -791,6 +807,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -804,6 +821,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -839,7 +857,7 @@ }, "gridPos": { "h": 7, - "w": 5, + "w": 11, "x": 0, "y": 16 }, @@ -883,6 +901,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -896,6 +915,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -931,11 +951,11 @@ }, "gridPos": { "h": 7, - "w": 6, - "x": 5, + "w": 13, + "x": 11, "y": 16 }, - "id": 33, + "id": 16, "options": { "legend": { "calcs": [], @@ -955,13 +975,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (node) (rate(fluentbit_input_bytes_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\", name=~\"azmon_tenant_${namespace}_.*\"}[5m]))", - "legendFormat": "{{node}}", + "expr": "sum by () (rate(fluentbit_output_proc_bytes_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\", name=~\"azmon_tenant_${namespace}_.*\"}[5m]))", + "legendFormat": "All", "range": true, "refId": "A" } ], - "title": "Fluent-bit input bytes/s by Node", + "title": "Fluent-bit output forwarded bytes/s by All Nodes", "type": "timeseries" }, { @@ -975,6 +995,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -988,6 +1009,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1022,12 +1044,12 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 5, - "x": 11, - "y": 16 + "h": 6, + "w": 11, + "x": 0, + "y": 23 }, - "id": 16, + "id": 33, "options": { "legend": { "calcs": [], @@ -1047,13 +1069,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by () (rate(fluentbit_output_proc_bytes_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\", name=~\"azmon_tenant_${namespace}_.*\"}[5m]))", - "legendFormat": "All", + "expr": "sum by (node) (rate(fluentbit_input_bytes_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\", name=~\"azmon_tenant_${namespace}_.*\"}[5m]))", + "legendFormat": "{{node}}", "range": true, "refId": "A" } ], - "title": "Fluent-bit output process bytes/s by All Nodes", + "title": "Fluent-bit input bytes/s by Node", "type": "timeseries" }, { @@ -1067,6 +1089,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1080,6 +1103,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1114,10 +1138,10 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 16 + "h": 6, + "w": 13, + "x": 11, + "y": 23 }, "id": 12, "options": { @@ -1145,7 +1169,7 @@ "refId": "A" } ], - "title": "Fluent-bit output process bytes/s by Node", + "title": "Fluent-bit output forwarded bytes/s by Node", "type": "timeseries" }, { @@ -1159,6 +1183,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1172,6 +1197,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1206,10 +1232,10 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 6, "w": 11, "x": 0, - "y": 23 + "y": 29 }, "id": 20, "options": { @@ -1237,7 +1263,7 @@ "refId": "A" } ], - "title": "Fluent-bit retries/fails", + "title": "Fluent-bit retries failed/sec by Node", "type": "timeseries" }, { @@ -1251,6 +1277,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1264,6 +1291,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1298,10 +1326,10 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 6, "w": 13, "x": 11, - "y": 23 + "y": 29 }, "id": 22, "options": { @@ -1323,25 +1351,108 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (node) (rate(fluentbit_output_errors_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\", name=~\"azmon_tenant_${namespace}_.*\"}[5m]))", + "expr": "sum by (node) (rate(fluentbit_output_errors_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\"}[5m]))", "legendFormat": "{{node}}", "range": true, "refId": "A" } ], - "title": "Fluent-bit output errors/sec", + "title": "Fluent-bit output errors/sec by Node", "type": "timeseries" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, + "h": 7, + "w": 8, "x": 0, - "y": 28 + "y": 35 }, - "id": 28, - "title": "Daemonset (all namespaces)", - "type": "row" + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by (node) (rate(fluentbit_input_files_opened_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\", name=~\"azmon_tenant_${namespace}_.*\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit input files opened/sec by Node", + "type": "timeseries" }, { "datasource": { @@ -1354,6 +1465,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1367,6 +1479,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1402,11 +1515,11 @@ }, "gridPos": { "h": 7, - "w": 5, - "x": 0, - "y": 29 + "w": 7, + "x": 8, + "y": 35 }, - "id": 10, + "id": 48, "options": { "legend": { "calcs": [], @@ -1426,13 +1539,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (node) (rate(fluentbit_filter_records_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\",name!~\"ama-logs-*|mon-agent-*|telegraf-*|azmon_tenant_record_modifier|azmon_tenant_grep|fluentbit_metrics_input|storage_*|fluentbit_metrics_input|oms_output\"}[5m]))", + "expr": "\r\nsum by (node) (rate(fluentbit_input_files_closed_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\", name=~\"azmon_tenant_${namespace}_.*\"}[5m]))", "legendFormat": "{{node}}", "range": true, "refId": "A" } ], - "title": "Fluent-bit incoming(before throttle) records/s by Node", + "title": "Fluent-bit input files closed/sec by Node", "type": "timeseries" }, { @@ -1446,6 +1559,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1459,6 +1573,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1494,11 +1609,116 @@ }, "gridPos": { "h": 7, - "w": 5, - "x": 5, - "y": 29 + "w": 9, + "x": 15, + "y": 35 }, - "id": 40, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by (node) (rate(fluentbit_input_files_rotated_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\", name=~\"azmon_tenant_${namespace}_.*\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit input files rotated/sec by Node", + "type": "timeseries" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 28, + "title": "Daemonset (all namespaces)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 43 + }, + "id": 10, "options": { "legend": { "calcs": [], @@ -1538,6 +1758,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1551,6 +1772,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1585,10 +1807,10 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 6, - "x": 11, - "y": 29 + "h": 8, + "w": 5, + "x": 6, + "y": 43 }, "id": 35, "options": { @@ -1630,6 +1852,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1643,6 +1866,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1677,10 +1901,10 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 7, - "x": 17, - "y": 29 + "h": 8, + "w": 13, + "x": 11, + "y": 43 }, "id": 36, "options": { @@ -1712,17 +1936,98 @@ "type": "timeseries" }, { - "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, + "h": 7, + "w": 11, "x": 0, - "y": 36 + "y": 51 }, - "id": 27, - "panels": [], - "title": "Service side (for all namespaces) (To be fixed)", - "type": "row" + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by (node) (rate(fluentbit_output_retries_failed_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit retries failed/sec by Node", + "type": "timeseries" }, { "datasource": { @@ -1735,6 +2040,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1748,6 +2054,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1783,11 +2090,11 @@ }, "gridPos": { "h": 7, - "w": 6, - "x": 0, - "y": 37 + "w": 13, + "x": 11, + "y": 51 }, - "id": 34, + "id": 46, "options": { "legend": { "calcs": [], @@ -1807,15 +2114,28 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (instance) (rate(fluentbit_input_records_total{cluster=~\"$cluster\", job=\"ama-logs-multitenancy\", instance=~\"ama-logs-multitenancy-.*\", name=\"input_ama-logs-service_forward\"}[5m]))", + "expr": "sum by (node) (rate(fluentbit_output_errors_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\" }[5m]))", "legendFormat": "{{node}}", "range": true, "refId": "A" } ], - "title": "Fluent-bit received records/s by Instance", + "title": "Fluent-bit output errors/sec by Node", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 27, + "panels": [], + "title": "Service side (for all namespaces)", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -1827,6 +2147,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1840,6 +2161,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1875,9 +2197,9 @@ }, "gridPos": { "h": 7, - "w": 6, - "x": 6, - "y": 37 + "w": 11, + "x": 0, + "y": 59 }, "id": 43, "options": { @@ -1919,6 +2241,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1932,6 +2255,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1967,9 +2291,9 @@ }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 37 + "w": 13, + "x": 11, + "y": 59 }, "id": 42, "options": { @@ -2011,6 +2335,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2024,6 +2349,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2058,10 +2384,10 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 6, + "h": 6, + "w": 11, "x": 0, - "y": 44 + "y": 66 }, "id": 41, "options": { @@ -2103,6 +2429,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2116,6 +2443,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2150,10 +2478,10 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 44 + "h": 6, + "w": 13, + "x": 11, + "y": 66 }, "id": 44, "options": { @@ -2187,8 +2515,7 @@ ], "refresh": "", "revision": 1, - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [ @@ -2196,7 +2523,7 @@ "current": { "selected": false, "text": "Managed_Prometheus_defaultazuremonitorworkspace-eus", - "value": "Managed_Prometheus_defaultazuremonitorworkspace-eus" + "value": "defaultazuremonitorworkspace-eus" }, "hide": 0, "includeAll": false, @@ -2237,9 +2564,9 @@ }, { "current": { - "selected": true, - "text": "app-team-27", - "value": "app-team-27" + "selected": false, + "text": "app-team-1", + "value": "app-team-1" }, "datasource": { "type": "prometheus", @@ -2270,8 +2597,8 @@ }, "timepicker": {}, "timezone": "", - "title": "Azure Monitor for Containers - Logs Multitenacy", + "title": "Azure Monitor / Container Insights /QoS - Logs Multitenacy - Advanced Mode", "uid": "E_FvdCC4z", - "version": 15, + "version": 2, "weekStart": "" } \ No newline at end of file diff --git a/Documentation/MultiTenancyLogging/ama-metrics-prometheus-config.yaml b/Documentation/MultiTenancyLogging/AdvancedMode/ama-metrics-prometheus-config-node.yaml similarity index 57% rename from Documentation/MultiTenancyLogging/ama-metrics-prometheus-config.yaml rename to Documentation/MultiTenancyLogging/AdvancedMode/ama-metrics-prometheus-config-node.yaml index 66b4865ce..028638c78 100644 --- a/Documentation/MultiTenancyLogging/ama-metrics-prometheus-config.yaml +++ b/Documentation/MultiTenancyLogging/AdvancedMode/ama-metrics-prometheus-config-node.yaml @@ -24,27 +24,9 @@ data: - source_labels: [__meta_kubernetes_pod_node_name] action: replace target_label: node - - job_name: ama-logs-multitenancy - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_pod_controller_kind] - action: keep - regex: 'ReplicaSet' - - source_labels: [__meta_kubernetes_pod_controller_name] - regex: '^ama-logs-multitenancy(-.*)?$' - action: keep - - source_labels: [__address__] - action: replace - target_label: __address__ - regex: (.+?)(\:\d+)? - replacement: $1:9102 - - source_labels: [__meta_kubernetes_pod_name] - action: replace - target_label: instance - source_labels: [__meta_kubernetes_pod_node_name] - action: replace - target_label: node + action: keep + regex: $NODE_NAME metadata: - name: ama-metrics-prometheus-config + name: ama-metrics-prometheus-config-node namespace: kube-system diff --git a/Documentation/MultiTenancyLogging/AdvancedMode/ama-metrics-prometheus-config.yaml b/Documentation/MultiTenancyLogging/AdvancedMode/ama-metrics-prometheus-config.yaml new file mode 100644 index 000000000..f595fd5a9 --- /dev/null +++ b/Documentation/MultiTenancyLogging/AdvancedMode/ama-metrics-prometheus-config.yaml @@ -0,0 +1,29 @@ +kind: ConfigMap +apiVersion: v1 +data: + prometheus-config: |- + scrape_configs: + - job_name: ama-logs-multitenancy + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_controller_kind] + action: keep + regex: 'ReplicaSet' + - source_labels: [__meta_kubernetes_pod_controller_name] + regex: '^ama-logs-multitenancy(-.*)?$' + action: keep + - source_labels: [__address__] + action: replace + target_label: __address__ + regex: (.+?)(\:\d+)? + replacement: $1:9102 + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: instance + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: node +metadata: + name: ama-metrics-prometheus-config + namespace: kube-system diff --git a/Documentation/MultiTenancyLogging/BasicMode/AzureMonitorContainers_BasicMode_Grafana.json b/Documentation/MultiTenancyLogging/BasicMode/AzureMonitorContainers_BasicMode_Grafana.json new file mode 100644 index 000000000..d8cc05218 --- /dev/null +++ b/Documentation/MultiTenancyLogging/BasicMode/AzureMonitorContainers_BasicMode_Grafana.json @@ -0,0 +1,1514 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 258, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 28, + "title": "Daemonset (all namespaces)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (node) (rate(fluentbit_input_records_total{cluster=~\"$cluster\", name!~\"ama-logs-*|mon-agent-*|telegraf-*|azmon_tenant_record_modifier|azmon_tenant_grep|fluentbit_metrics_input|storage_*|fluentbit_metrics_input|oms_output\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit input records/s by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 8, + "y": 1 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (node) (rate(fluentbit_output_dropped_records_total{cluster=~\"$cluster\", name=~\"oms_output\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit dropped records/s by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 15, + "y": 1 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (node) (rate(fluentbit_output_proc_records_total{cluster=~\"$cluster\",name!~\"ama-logs-*|mon-agent-*|telegraf-*|azmon_tenant_record_modifier|azmon_tenant_grep|fluentbit_metrics_input|storage_*|fluentbit_metrics_input\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit output records/s by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 0, + "y": 9 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (node) (rate(fluentbit_input_bytes_total{cluster=~\"$cluster\", name!~\"ama-logs-*|mon-agent-*|telegraf-*|azmon_tenant_record_modifier|azmon_tenant_grep|fluentbit_metrics_input|storage_*|fluentbit_metrics_input|oms_output\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit input bytes/s by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 11, + "y": 9 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (node) (rate(fluentbit_output_proc_bytes_total{cluster=~\"$cluster\",name!~\"ama-logs-*|mon-agent-*|telegraf-*|azmon_tenant_record_modifier|azmon_tenant_grep|fluentbit_metrics_input|storage_*|fluentbit_metrics_input\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit output bytes/s by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by () (rate(fluentbit_input_records_total{cluster=~\"$cluster\", name!~\"ama-logs-*|mon-agent-*|telegraf-*|azmon_tenant_record_modifier|azmon_tenant_grep|fluentbit_metrics_input|storage_*|fluentbit_metrics_input|oms_output\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit input records/s by All Nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 8, + "y": 17 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by () (rate(fluentbit_output_dropped_records_total{cluster=~\"$cluster\", name=~\"oms_output\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit dropped records/s by All Nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 15, + "y": 17 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (node) (rate(fluentbit_output_proc_records_total{cluster=~\"$cluster\",name!~\"ama-logs-*|mon-agent-*|telegraf-*|azmon_tenant_record_modifier|azmon_tenant_grep|fluentbit_metrics_input|storage_*|fluentbit_metrics_input\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit output records/s by All Nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 25 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by (node) (rate(fluentbit_output_retries_failed_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit retries failed/sec by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 11, + "y": 25 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by (node) (rate(fluentbit_output_errors_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit output errors/sec by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 32 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by () (rate(fluentbit_output_retries_failed_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit retries failed/sec by All Nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 11, + "y": 32 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by () (rate(fluentbit_output_errors_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit output errors/sec by All Nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 39 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by (node) (rate(fluentbit_input_files_rotated_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit rotated files/sec by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 11, + "y": 39 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by (node) (rate(fluentbit_input_files_opened_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit input opened files/sec by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 0, + "y": 46 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\r\nsum by (node) (rate(fluentbit_input_files_closed_total{cluster=~\"$cluster\", job=\"ama-logs-daemonset\"}[5m]))", + "legendFormat": "{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Fluent-bit input closed files/sec by Node", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Managed_Prometheus_defaultazuremonitorworkspace-eus", + "value": "defaultazuremonitorworkspace-eus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "gangams-aks-log-scale-neu", + "value": "gangams-aks-log-scale-neu" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(up{job=\"cadvisor\"}, cluster)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(up{job=\"cadvisor\"}, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Azure Monitor / Container Insights /QoS - Logs Multitenacy - Basic Mode", + "uid": "E_FvdCC4z", + "version": 7, + "weekStart": "" +} \ No newline at end of file diff --git a/Documentation/MultiTenancyLogging/BasicMode/ama-metrics-prometheus-config-node.yaml b/Documentation/MultiTenancyLogging/BasicMode/ama-metrics-prometheus-config-node.yaml new file mode 100644 index 000000000..028638c78 --- /dev/null +++ b/Documentation/MultiTenancyLogging/BasicMode/ama-metrics-prometheus-config-node.yaml @@ -0,0 +1,32 @@ +kind: ConfigMap +apiVersion: v1 +data: + prometheus-config: |- + scrape_configs: + - job_name: ama-logs-daemonset + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_controller_kind] + action: keep + regex: 'DaemonSet' + - source_labels: [__meta_kubernetes_pod_controller_name] + regex: ^(ama-logs|ama-logs-windows)$ + action: keep + - source_labels: [__address__] + action: replace + target_label: __address__ + regex: (.+?)(\:\d+)? + replacement: $1:9102 + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: instance + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: node + - source_labels: [__meta_kubernetes_pod_node_name] + action: keep + regex: $NODE_NAME +metadata: + name: ama-metrics-prometheus-config-node + namespace: kube-system diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index 3b6892419..3e747fda7 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -31,6 +31,7 @@ @annotationBasedLogFiltering = false @allowed_system_namespaces = ['kube-system', 'gatekeeper-system', 'calico-system', 'azure-arc', 'kube-public', 'kube-node-lease'] @isAzMonMultiTenancyLogCollectionEnabled = false +@isAzMonMultiTenancyFallbackIngestionDisabled = false @isAzMonMultiTenancyLogCollectionAdvancedMode = false @azMonMultiTenantNamespaces = [] @azMonMultiTenancyMaxStorageChunksUp = 500 @@ -468,6 +469,10 @@ def populateSettingValuesFromConfigMap(parsedConfig) multi_tenancy_enabled = parsedConfig[:log_collection_settings][:multi_tenancy][:enabled] if multi_tenancy_enabled @isAzMonMultiTenancyLogCollectionEnabled = multi_tenancy_enabled + if !parsedConfig[:log_collection_settings][:multi_tenancy][:disable_fallback_ingestion].nil? + @isAzMonMultiTenancyFallbackIngestionDisabled = parsedConfig[:log_collection_settings][:multi_tenancy][:disable_fallback_ingestion] + end + # multi-tenancy advanced mode if !parsedConfig[:log_collection_settings][:multi_tenancy][:advanced_mode_enabled].nil? advanced_mode_enabled = parsedConfig[:log_collection_settings][:multi_tenancy][:advanced_mode_enabled] @@ -610,6 +615,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) file.write("export AZMON_ANNOTATION_BASED_LOG_FILTERING=#{@annotationBasedLogFiltering}\n") if @isAzMonMultiTenancyLogCollectionEnabled file.write("export AZMON_MULTI_TENANCY_LOG_COLLECTION=#{@isAzMonMultiTenancyLogCollectionEnabled}\n") + file.write("export AZMON_MULTI_TENANCY_FALLBACK_INGESTION_DISABLED=#{@isAzMonMultiTenancyFallbackIngestionDisabled}\n") file.write("export AZMON_MULTI_TENANCY_LOG_COLLECTION_ADVANCED_MODE=#{@isAzMonMultiTenancyLogCollectionAdvancedMode}\n") azMonMultiTenantNamespacesString = @azMonMultiTenantNamespaces.join(",") file.write("export AZMON_MULTI_TENANCY_NAMESPACES=#{azMonMultiTenantNamespacesString}\n") @@ -695,6 +701,8 @@ def get_command_windows(env_variable_name, env_variable_value) if @isAzMonMultiTenancyLogCollectionEnabled commands = get_command_windows("AZMON_MULTI_TENANCY_LOG_COLLECTION", @isAzMonMultiTenancyLogCollectionEnabled) file.write(commands) + commands = get_command_windows("AZMON_MULTI_TENANCY_FALLBACK_INGESTION_DISABLED", @isAzMonMultiTenancyFallbackIngestionDisabled) + file.write(commands) commands = get_command_windows("AZMON_MULTI_TENANCY_LOG_COLLECTION_ADVANCED_MODE", @isAzMonMultiTenancyLogCollectionAdvancedMode) file.write(commands) azMonMultiTenantNamespacesString = @azMonMultiTenantNamespaces.join(",") diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index 75aa1d6b4..dc960f470 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -12,14 +12,16 @@ data: # Any errors related to config map settings can be found in the KubeMonAgentEvents table in the Log Analytics workspace that the cluster is sending data to. [log_collection_settings] - # This feature in private preview and not ready for production use,please reach out us to try out this feature. + ## This feature in private preview and not ready for production use,please reach out us to try out this feature. # [log_collection_settings.multi_tenancy] - # # High log scale MUST be enabled to use this feature. Refer to https://aka.ms/cihsmode for more details on high log scale mode + ## High log scale MUST be enabled to use this feature. Refer to https://aka.ms/cihsmode for more details on high log scale mode # enabled = true + # disable_fallback_ingestion = false # by default fallback ingestion to default container insights extension DCR is enabled ### Below advanced settings MUST be provided when ama-logs-multitenancy service deployed through AKS Monitoring Addon + ### Advanced mode recommened when the log scale more than the 50K logs/sec per node ## advanced_mode_enabled = false # Default value is false - ## namespaces = ["app-team-1","app-team-2"] - ## namespace_settings = ["app-team-1:throttle_rate=2000","app-team-2:throttle_rate=5000"] + ## namespaces = ["app-team-1","app-team-2"] # All namespaces MUST be provided when advanced_mode_enabled is set to true since there will be separate fluent-bit plugins for each namespace + ## namespace_settings = ["app-team-1:throttle_rate=2000","app-team-2:throttle_rate=5000"] # optional settings for each namespace if the default settings are not sufficient [log_collection_settings.stdout] # In the absense of this configmap, default value for enabled is true @@ -76,7 +78,7 @@ data: # if include_fields commented out or empty, all fields will be included. If include_fields is set, only the fields listed will be included. # include_fields = ["podLabels","podAnnotations","podUid","image","imageID","imageRepo","imageTag"] #[log_collection_settings.filter_using_annotations] - # if enabled will exclude logs from pods with annotations fluenbit.io/exclude: "true". + # if enabled will exclude logs from pods with annotations fluentbit.io/exclude: "true". # Read more: https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#kubernetes-annotations # enabled = false diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index f9ed62ffa..5d71a29f5 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -205,6 +205,8 @@ var ( InsightsMetricsNamedPipe net.Conn // flag to check whether Azure Monitor Multi-tenancy Log Collection enabled or not IsAzMonMultiTenancyLogCollectionEnabled bool + // flag to check whether Azure Monitor Multi-tenancy Fallback ingestion disabled or not + IsAzMonMultiTenancyFallbackIngestionDisabled bool // flag to check whether Azure Monitor Multi-tenancy Log Collection Advanced Mode enabled or not IsAzMonMultiTenancyLogCollectionAdvancedModeEnabled bool // flag to check whether Azure Monitor Multi-tenancy Logs ServiceMode enabled or not @@ -1957,8 +1959,7 @@ func writeMsgPackEntries(connection net.Conn, isContainerLogV2Schema bool, fluen totalBytes := 0 for namespace, entries := range msgPackEntriesByNamespace { if streamTags, exists := namespaceStreamIdsMap[namespace]; exists { - msg := fmt.Sprintf("Info::ama:: namespace : %s streamTags: %s \n", namespace, strings.Join(streamTags, ", ")) - Log(msg) + Log("Info::ama:: namespace : %s streamTags: %s \n", namespace, strings.Join(streamTags, ", ")) for _, streamTag := range streamTags { if IsWindows { bts, er = writeMsgPackEntriesToNamedPipeConnection(streamTag, entries, streamIdNamedPipeMap) @@ -1969,20 +1970,43 @@ func writeMsgPackEntries(connection net.Conn, isContainerLogV2Schema bool, fluen bts, er = connection.Write(msgpBytes) } if er != nil { - return bts, er + return totalBytes, er } - totalBytes = totalBytes + bts + totalBytes += bts } } else { - Log("Info::ama:: streamTag is empty for namespace: %s hence using default workspace stream id: %s \n", namespace, fluentForwardTag) - msgpBytes := convertMsgPackEntriesToMsgpBytes(fluentForwardTag, entries) - deadline := 10 * time.Second - connection.SetWriteDeadline(time.Now().Add(deadline)) - bts, er = connection.Write(msgpBytes) + if IsAzMonMultiTenancyFallbackIngestionDisabled { + Log("Info::ama:: streamTag is empty for namespace: %s and fallback ingestion disabled hence skipping the logs to default container insights extension stream \n", namespace) + continue + } else { + Log("Info::ama:: streamTag is empty for namespace: %s hence using default workspace stream id: %s \n", namespace, fluentForwardTag) + msgpBytes := convertMsgPackEntriesToMsgpBytes(fluentForwardTag, entries) + deadline := 10 * time.Second + connection.SetWriteDeadline(time.Now().Add(deadline)) + bts, er = connection.Write(msgpBytes) + if er != nil { + return totalBytes, er + } + totalBytes += bts + } + } + } + // handle if DCR has _ALL_K8S_NAMESPACES_ + if streamTags, exists := namespaceStreamIdsMap["_all_k8s_namespaces_"]; exists { + Log("Info::ama:: namespace : _All_K8S_NAMESPACES_ streamTags: %s \n", strings.Join(streamTags, ", ")) + for _, streamTag := range streamTags { + if IsWindows { + bts, er = writeMsgPackEntriesToNamedPipeConnection(streamTag, msgPackEntries, streamIdNamedPipeMap) + } else { + msgpBytes := convertMsgPackEntriesToMsgpBytes(streamTag, msgPackEntries) + deadline := 10 * time.Second + connection.SetWriteDeadline(time.Now().Add(deadline)) + bts, er = connection.Write(msgpBytes) + } if er != nil { - return bts, er + return totalBytes, er } - totalBytes = totalBytes + bts + totalBytes += bts } } bts = totalBytes @@ -2344,6 +2368,13 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { Log("Azure Monitor Multi-tenancy Log Collection Enabled") } + IsAzMonMultiTenancyFallbackIngestionDisabled = false + multiTenancyFallbackIngestionDisabled := strings.TrimSpace(strings.ToLower(os.Getenv("AZMON_MULTI_TENANCY_FALLBACK_INGESTION_DISABLED"))) + if multiTenancyFallbackIngestionDisabled != "" && strings.Compare(strings.ToLower(multiTenancyFallbackIngestionDisabled), "true") == 0 { + IsAzMonMultiTenancyFallbackIngestionDisabled = true + Log("Azure Monitor Multi-tenancy Fallback ingestion disabled") + } + IsAzMonMultiTenancyLogCollectionAdvancedModeEnabled = false multiTenancyAdvancedModeEnabled := strings.TrimSpace(strings.ToLower(os.Getenv("AZMON_MULTI_TENANCY_LOG_COLLECTION_ADVANCED_MODE"))) if multiTenancyAdvancedModeEnabled != "" && strings.Compare(strings.ToLower(multiTenancyAdvancedModeEnabled), "true") == 0 { diff --git a/source/plugins/go/src/telemetry.go b/source/plugins/go/src/telemetry.go index 08a7a2016..689a6c402 100644 --- a/source/plugins/go/src/telemetry.go +++ b/source/plugins/go/src/telemetry.go @@ -279,11 +279,13 @@ func SendContainerLogPluginMetrics(telemetryPushIntervalProperty string) { isAzMonMultitenancyEnabled := os.Getenv("AZMON_MULTI_TENANCY_LOG_COLLECTION") isAzMonMultitenancyAdvancedMode := os.Getenv("AZMON_MULTI_TENANCY_LOG_COLLECTION_ADVANCED_MODE") + isAzMonMultitenancyDefaultFallbackIngestionDisabled := os.Getenv("AZMON_MULTI_TENANCY_FALLBACK_INGESTION_DISABLED") if isAzMonMultitenancyEnabled != "" { telemetryDimensions["isAzMonMultitenancyEnabled"] = isAzMonMultitenancyEnabled telemetryDimensions["isAzMonMultitenancyAdvancedMode"] = isAzMonMultitenancyAdvancedMode telemetryDimensions["containerLogV2ExtensionDCRCount"] = strconv.Itoa(containerLogV2ExtensionDCRCount) telemetryDimensions["multitenantNamespaceCount"] = strconv.Itoa(multitenantNamespaceCount) + telemetryDimensions["isAzMonMultitenancyFallbackIngestionDisabled"] = isAzMonMultitenancyDefaultFallbackIngestionDisabled } enableCustomMetrics := os.Getenv("ENABLE_CUSTOM_METRICS")