From 59564e70e6754899453c8c9bc4ac016627851cef Mon Sep 17 00:00:00 2001 From: Simone Rodigari Date: Tue, 21 Jan 2025 13:43:06 +0000 Subject: [PATCH 1/3] feat(conntrack-metrics): legacy control plane basic mode --- pkg/metrics/metrics.go | 28 +++++++++++++++++++ pkg/metrics/types.go | 12 ++++++++ pkg/plugin/packetparser/packetparser_linux.go | 23 +++++++++++++++ pkg/utils/attr_utils.go | 3 ++ pkg/utils/metric_names.go | 6 ++++ 5 files changed, 72 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index fde18e4ab6..cbb387ad0b 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -158,6 +158,34 @@ func InitializeMetrics() { utils.InterfaceName, ) + ConntrackPacketsForward = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackPacketsForwardGaugeName, + ConntrackPacketForwardDescription, + utils.ConntrackGaugeLabels..., + ) + + ConntrackPacketsReply = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackPacketsReplyGaugeName, + ConntrackPacketReplyDescription, + utils.ConntrackGaugeLabels..., + ) + + ConntrackBytesForward = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackBytesForwardGaugeName, + ConntrackBytesForwardDescription, + utils.ConntrackGaugeLabels..., + ) + + ConntrackBytesReply = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackBytesReplyGaugeName, + ConntrackBytesReplyDescription, + utils.ConntrackGaugeLabels..., + ) + isInitialized = true metricsLogger.Info("Metrics initialized") } diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go index cff5c28884..1f4e2d0a66 100644 --- a/pkg/metrics/types.go +++ b/pkg/metrics/types.go @@ -43,6 +43,12 @@ const ( // Control plane metrics pluginManagerFailedToReconcileCounterDescription = "Number of times the plugin manager failed to reconcile the plugins" lostEventsCounterDescription = "Number of events lost in control plane" + + // Conntrack metrics + ConntrackPacketForwardDescription = "Number of forward packets" + ConntrackPacketReplyDescription = "Number of reply packets" + ConntrackBytesForwardDescription = "Number of forward bytes" + ConntrackBytesReplyDescription = "Number of reply bytes" ) // Metric Counters @@ -89,6 +95,12 @@ var ( InfinibandStatsGauge GaugeVec InfinibandStatusParamsGauge GaugeVec + + // Conntrack + ConntrackPacketsForward GaugeVec + ConntrackPacketsReply GaugeVec + ConntrackBytesForward GaugeVec + ConntrackBytesReply GaugeVec ) func ToPrometheusType(metric interface{}) prometheus.Collector { diff --git a/pkg/plugin/packetparser/packetparser_linux.go b/pkg/plugin/packetparser/packetparser_linux.go index fb54c8bb5e..d2cc22a1c8 100644 --- a/pkg/plugin/packetparser/packetparser_linux.go +++ b/pkg/plugin/packetparser/packetparser_linux.go @@ -637,6 +637,19 @@ func (p *packetParser) processRecord(ctx context.Context, id int) { p.enricher.Write(ev) } + // Add conntrack metrics. + if p.cfg.EnableConntrackMetrics { + labels := []string{ + protoToString(bpfEvent.Proto), + fl.GetTrafficDirection().String(), + } + // Basic metrics, node-level + metrics.ConntrackPacketsForward.WithLabelValues(labels...).Set(float64(bpfEvent.ConntrackMetadata.PacketsForwardCount)) + metrics.ConntrackBytesForward.WithLabelValues(labels...).Set(float64(bpfEvent.ConntrackMetadata.BytesForwardCount)) + metrics.ConntrackPacketsReply.WithLabelValues(labels...).Set(float64(bpfEvent.ConntrackMetadata.PacketsReplyCount)) + metrics.ConntrackBytesReply.WithLabelValues(labels...).Set(float64(bpfEvent.ConntrackMetadata.BytesReplyCount)) + } + // Write the event to the external channel. if p.externalChannel != nil { select { @@ -704,3 +717,13 @@ func absPath() (string, error) { dir := path.Dir(filename) return dir, nil } + +func protoToString(bpfEventProto uint8) string { + var proto string + if bpfEventProto == 6 { + proto = "tcp" + } else if bpfEventProto == 17 { + proto = "udp" + } + return proto +} diff --git a/pkg/utils/attr_utils.go b/pkg/utils/attr_utils.go index 6826db74c9..1765b14c7e 100644 --- a/pkg/utils/attr_utils.go +++ b/pkg/utils/attr_utils.go @@ -85,6 +85,9 @@ var ( // DNS labels. DNSRequestLabels = []string{"query_type", "query"} DNSResponseLabels = []string{"return_code", "query_type", "query", "response", "num_response"} + + // Flow labels. + ConntrackGaugeLabels = []string{"protocol", "traffic_direction"} ) func GetPluginEventAttributes(attrs []attribute.KeyValue, pluginName, eventName, timestamp string) []attribute.KeyValue { diff --git a/pkg/utils/metric_names.go b/pkg/utils/metric_names.go index 8060b05243..bd5a3339e1 100644 --- a/pkg/utils/metric_names.go +++ b/pkg/utils/metric_names.go @@ -34,6 +34,12 @@ const ( // Common Gauges across os distributions NodeConnectivityStatusName = "node_connectivity_status" NodeConnectivityLatencySecondsName = "node_connectivity_latency_seconds" + + // Conntrack + ConntrackPacketsForwardGaugeName = "conntrack_packets_forward" + ConntrackPacketsReplyGaugeName = "conntrack_packets_reply" + ConntrackBytesForwardGaugeName = "conntrack_bytes_forward" + ConntrackBytesReplyGaugeName = "conntrack_bytes_reply" ) // IsAdvancedMetric is a helper function to determine if a name is an advanced metric From 191464d398e5c1609b871a9bf426239e447090bc Mon Sep 17 00:00:00 2001 From: Simone Rodigari Date: Tue, 21 Jan 2025 16:49:45 +0000 Subject: [PATCH 2/3] feat(conntrack-metrics): move implementation to conntrack gc, remove labels --- pkg/metrics/metrics.go | 4 --- pkg/plugin/conntrack/conntrack_linux.go | 32 +++++++++++++++++++ pkg/plugin/packetparser/packetparser_linux.go | 23 ------------- pkg/utils/attr_utils.go | 3 -- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index cbb387ad0b..17c2bae292 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -162,28 +162,24 @@ func InitializeMetrics() { exporter.DefaultRegistry, utils.ConntrackPacketsForwardGaugeName, ConntrackPacketForwardDescription, - utils.ConntrackGaugeLabels..., ) ConntrackPacketsReply = exporter.CreatePrometheusGaugeVecForMetric( exporter.DefaultRegistry, utils.ConntrackPacketsReplyGaugeName, ConntrackPacketReplyDescription, - utils.ConntrackGaugeLabels..., ) ConntrackBytesForward = exporter.CreatePrometheusGaugeVecForMetric( exporter.DefaultRegistry, utils.ConntrackBytesForwardGaugeName, ConntrackBytesForwardDescription, - utils.ConntrackGaugeLabels..., ) ConntrackBytesReply = exporter.CreatePrometheusGaugeVecForMetric( exporter.DefaultRegistry, utils.ConntrackBytesReplyGaugeName, ConntrackBytesReplyDescription, - utils.ConntrackGaugeLabels..., ) isInitialized = true diff --git a/pkg/plugin/conntrack/conntrack_linux.go b/pkg/plugin/conntrack/conntrack_linux.go index dc2fb0c449..bf0a44c870 100644 --- a/pkg/plugin/conntrack/conntrack_linux.go +++ b/pkg/plugin/conntrack/conntrack_linux.go @@ -16,6 +16,7 @@ import ( "github.com/microsoft/retina/internal/ktime" "github.com/microsoft/retina/pkg/loader" "github.com/microsoft/retina/pkg/log" + "github.com/microsoft/retina/pkg/metrics" plugincommon "github.com/microsoft/retina/pkg/plugin/common" _ "github.com/microsoft/retina/pkg/plugin/conntrack/_cprog" // nolint // This is needed so cprog is included when vendoring "github.com/microsoft/retina/pkg/utils" @@ -23,6 +24,8 @@ import ( "go.uber.org/zap" ) +var conntrackMetricsEnabled = false // global variable to enable conntrack metrics + //go:generate go run github.com/cilium/ebpf/cmd/bpf2go@master -cflags "-g -O2 -Wall -D__TARGET_ARCH_${GOARCH} -Wall" -target ${GOARCH} -type ct_v4_key conntrack ./_cprog/conntrack.c -- -I../lib/_${GOARCH} -I../lib/common/libbpf/_src -I../lib/common/libbpf/_include/linux -I../lib/common/libbpf/_include/uapi/linux -I../lib/common/libbpf/_include/asm // Init initializes the conntrack eBPF map in the kernel for the first time. @@ -88,6 +91,10 @@ func GenerateDynamic(ctx context.Context, dynamicHeaderPath string, conntrackMet if err != nil { return errors.Wrap(err, "failed to write conntrack dynamic header") } + // set a global variable to enable conntrack metrics + if conntrackMetrics == 1 { + conntrackMetricsEnabled = true + } return nil } @@ -118,6 +125,10 @@ func (ct *Conntrack) Run(ctx context.Context) error { // List of keys to be deleted var keysToDelete []conntrackCtV4Key + // metrics counters + var packetsCountForward, packetsCountReply uint32 + var bytesCountForward, bytesCountReply uint64 + iter := ct.ctMap.Iterate() for iter.Next(&key, &value) { noOfCtEntries++ @@ -133,6 +144,18 @@ func (ct *Conntrack) Run(ctx context.Context) error { dstIP := utils.Int2ip(key.DstIp).To4() sourcePortShort := uint32(utils.HostToNetShort(key.SrcPort)) destinationPortShort := uint32(utils.HostToNetShort(key.DstPort)) + + // Add conntrack metrics. + if conntrackMetricsEnabled { + // Basic metrics, node-level + // for each ct_entry increment counters + ctMeta := value.ConntrackMetadata + packetsCountForward += ctMeta.PacketsForwardCount + packetsCountReply += ctMeta.PacketsReplyCount + bytesCountForward += ctMeta.BytesForwardCount + bytesCountReply += ctMeta.BytesReplyCount + } + ct.l.Debug("conntrack entry", zap.String("src_ip", srcIP.String()), zap.Uint32("src_port", sourcePortShort), @@ -151,6 +174,15 @@ func (ct *Conntrack) Run(ctx context.Context) error { if err := iter.Err(); err != nil { ct.l.Error("Iterate failed", zap.Error(err)) } + + // create metrics + if conntrackMetricsEnabled { + metrics.ConntrackPacketsForward.WithLabelValues().Set(float64(packetsCountForward)) + metrics.ConntrackBytesForward.WithLabelValues().Set(float64(bytesCountForward)) + metrics.ConntrackPacketsReply.WithLabelValues().Set(float64(packetsCountReply)) + metrics.ConntrackBytesReply.WithLabelValues().Set(float64(bytesCountReply)) + } + // Delete the conntrack entries for _, key := range keysToDelete { if err := ct.ctMap.Delete(key); err != nil { diff --git a/pkg/plugin/packetparser/packetparser_linux.go b/pkg/plugin/packetparser/packetparser_linux.go index d2cc22a1c8..fb54c8bb5e 100644 --- a/pkg/plugin/packetparser/packetparser_linux.go +++ b/pkg/plugin/packetparser/packetparser_linux.go @@ -637,19 +637,6 @@ func (p *packetParser) processRecord(ctx context.Context, id int) { p.enricher.Write(ev) } - // Add conntrack metrics. - if p.cfg.EnableConntrackMetrics { - labels := []string{ - protoToString(bpfEvent.Proto), - fl.GetTrafficDirection().String(), - } - // Basic metrics, node-level - metrics.ConntrackPacketsForward.WithLabelValues(labels...).Set(float64(bpfEvent.ConntrackMetadata.PacketsForwardCount)) - metrics.ConntrackBytesForward.WithLabelValues(labels...).Set(float64(bpfEvent.ConntrackMetadata.BytesForwardCount)) - metrics.ConntrackPacketsReply.WithLabelValues(labels...).Set(float64(bpfEvent.ConntrackMetadata.PacketsReplyCount)) - metrics.ConntrackBytesReply.WithLabelValues(labels...).Set(float64(bpfEvent.ConntrackMetadata.BytesReplyCount)) - } - // Write the event to the external channel. if p.externalChannel != nil { select { @@ -717,13 +704,3 @@ func absPath() (string, error) { dir := path.Dir(filename) return dir, nil } - -func protoToString(bpfEventProto uint8) string { - var proto string - if bpfEventProto == 6 { - proto = "tcp" - } else if bpfEventProto == 17 { - proto = "udp" - } - return proto -} diff --git a/pkg/utils/attr_utils.go b/pkg/utils/attr_utils.go index 1765b14c7e..6826db74c9 100644 --- a/pkg/utils/attr_utils.go +++ b/pkg/utils/attr_utils.go @@ -85,9 +85,6 @@ var ( // DNS labels. DNSRequestLabels = []string{"query_type", "query"} DNSResponseLabels = []string{"return_code", "query_type", "query", "response", "num_response"} - - // Flow labels. - ConntrackGaugeLabels = []string{"protocol", "traffic_direction"} ) func GetPluginEventAttributes(attrs []attribute.KeyValue, pluginName, eventName, timestamp string) []attribute.KeyValue { From bcc9773ea7ff34fd94978c068717eb2722f3b3f5 Mon Sep 17 00:00:00 2001 From: Simone Rodigari Date: Tue, 21 Jan 2025 17:25:12 +0000 Subject: [PATCH 3/3] feat(conntrack-metrics): add conntrack_total_connections metric and fix comments --- pkg/metrics/metrics.go | 6 ++++++ pkg/metrics/types.go | 18 ++++++++++-------- pkg/plugin/conntrack/conntrack_linux.go | 9 +++++---- pkg/utils/metric_names.go | 1 + 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 17c2bae292..91d8201a33 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -182,6 +182,12 @@ func InitializeMetrics() { ConntrackBytesReplyDescription, ) + ConntrackTotalConnections = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackTotalConnectionsName, + ConntrackTotalConnectionsDescription, + ) + isInitialized = true metricsLogger.Info("Metrics initialized") } diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go index 1f4e2d0a66..d934d0338e 100644 --- a/pkg/metrics/types.go +++ b/pkg/metrics/types.go @@ -45,10 +45,11 @@ const ( lostEventsCounterDescription = "Number of events lost in control plane" // Conntrack metrics - ConntrackPacketForwardDescription = "Number of forward packets" - ConntrackPacketReplyDescription = "Number of reply packets" - ConntrackBytesForwardDescription = "Number of forward bytes" - ConntrackBytesReplyDescription = "Number of reply bytes" + ConntrackPacketForwardDescription = "Number of forward packets" + ConntrackPacketReplyDescription = "Number of reply packets" + ConntrackBytesForwardDescription = "Number of forward bytes" + ConntrackBytesReplyDescription = "Number of reply bytes" + ConntrackTotalConnectionsDescription = "Total number of connections" ) // Metric Counters @@ -97,10 +98,11 @@ var ( InfinibandStatusParamsGauge GaugeVec // Conntrack - ConntrackPacketsForward GaugeVec - ConntrackPacketsReply GaugeVec - ConntrackBytesForward GaugeVec - ConntrackBytesReply GaugeVec + ConntrackPacketsForward GaugeVec + ConntrackPacketsReply GaugeVec + ConntrackBytesForward GaugeVec + ConntrackBytesReply GaugeVec + ConntrackTotalConnections GaugeVec ) func ToPrometheusType(metric interface{}) prometheus.Collector { diff --git a/pkg/plugin/conntrack/conntrack_linux.go b/pkg/plugin/conntrack/conntrack_linux.go index bf0a44c870..fb61f33f9b 100644 --- a/pkg/plugin/conntrack/conntrack_linux.go +++ b/pkg/plugin/conntrack/conntrack_linux.go @@ -24,7 +24,7 @@ import ( "go.uber.org/zap" ) -var conntrackMetricsEnabled = false // global variable to enable conntrack metrics +var conntrackMetricsEnabled = false // conntrack metrics global variable //go:generate go run github.com/cilium/ebpf/cmd/bpf2go@master -cflags "-g -O2 -Wall -D__TARGET_ARCH_${GOARCH} -Wall" -target ${GOARCH} -type ct_v4_key conntrack ./_cprog/conntrack.c -- -I../lib/_${GOARCH} -I../lib/common/libbpf/_src -I../lib/common/libbpf/_include/linux -I../lib/common/libbpf/_include/uapi/linux -I../lib/common/libbpf/_include/asm @@ -91,7 +91,7 @@ func GenerateDynamic(ctx context.Context, dynamicHeaderPath string, conntrackMet if err != nil { return errors.Wrap(err, "failed to write conntrack dynamic header") } - // set a global variable to enable conntrack metrics + // set a global variable if conntrackMetrics == 1 { conntrackMetricsEnabled = true } @@ -126,7 +126,7 @@ func (ct *Conntrack) Run(ctx context.Context) error { var keysToDelete []conntrackCtV4Key // metrics counters - var packetsCountForward, packetsCountReply uint32 + var packetsCountForward, packetsCountReply, totConnections uint32 var bytesCountForward, bytesCountReply uint64 iter := ct.ctMap.Iterate() @@ -148,8 +148,8 @@ func (ct *Conntrack) Run(ctx context.Context) error { // Add conntrack metrics. if conntrackMetricsEnabled { // Basic metrics, node-level - // for each ct_entry increment counters ctMeta := value.ConntrackMetadata + totConnections++ packetsCountForward += ctMeta.PacketsForwardCount packetsCountReply += ctMeta.PacketsReplyCount bytesCountForward += ctMeta.BytesForwardCount @@ -181,6 +181,7 @@ func (ct *Conntrack) Run(ctx context.Context) error { metrics.ConntrackBytesForward.WithLabelValues().Set(float64(bytesCountForward)) metrics.ConntrackPacketsReply.WithLabelValues().Set(float64(packetsCountReply)) metrics.ConntrackBytesReply.WithLabelValues().Set(float64(bytesCountReply)) + metrics.ConntrackTotalConnections.WithLabelValues().Set(float64(totConnections)) } // Delete the conntrack entries diff --git a/pkg/utils/metric_names.go b/pkg/utils/metric_names.go index bd5a3339e1..6d949dc8a0 100644 --- a/pkg/utils/metric_names.go +++ b/pkg/utils/metric_names.go @@ -40,6 +40,7 @@ const ( ConntrackPacketsReplyGaugeName = "conntrack_packets_reply" ConntrackBytesForwardGaugeName = "conntrack_bytes_forward" ConntrackBytesReplyGaugeName = "conntrack_bytes_reply" + ConntrackTotalConnectionsName = "conntrack_total_connections" ) // IsAdvancedMetric is a helper function to determine if a name is an advanced metric