From 393fe33d06415af2cc78f31113b4149f38c59f94 Mon Sep 17 00:00:00 2001 From: beegiik Date: Wed, 20 Nov 2024 16:24:48 +0000 Subject: [PATCH 1/3] feat(windows): Add pod level labels for windows HNS metric --- pkg/enricher/enricher.go | 29 +++++++++++ .../windows/hnsstats/hnsstats_windows.go | 51 +++++++++++++++++++ pkg/plugin/windows/hnsstats/types_windows.go | 16 ++++++ pkg/utils/attr_utils.go | 7 +++ 4 files changed, 103 insertions(+) diff --git a/pkg/enricher/enricher.go b/pkg/enricher/enricher.go index 98013cd2ca..7155bce253 100644 --- a/pkg/enricher/enricher.go +++ b/pkg/enricher/enricher.go @@ -14,6 +14,7 @@ import ( "github.com/microsoft/retina/pkg/common" "github.com/microsoft/retina/pkg/controllers/cache" "github.com/microsoft/retina/pkg/log" + "github.com/microsoft/retina/pkg/utils" "go.uber.org/zap" ) @@ -182,6 +183,34 @@ func (e *Enricher) getWorkloads(ownerRefs []*common.OwnerReference) []*flow.Work return workloads } +func (e *Enricher) GetWindowLabels(ip string) *utils.LabelsInfo { + obj := e.cache.GetObjByIP(ip) + if obj == nil { + e.l.Debug("No object found for IP", zap.String("ip", ip)) + return nil + } + + switch o := obj.(type) { + case *common.RetinaEndpoint: + var workload *flow.Workload + if workloads := e.getWorkloads(o.OwnerRefs()); len(workloads) > 0 { + workload = workloads[0] + } else { + workload = &flow.Workload{Name: "", Kind: ""} + } + + return &utils.LabelsInfo{ + Namespace: o.Namespace(), + PodName: o.Name(), + Workloads: workload, + } + + default: + e.l.Debug("received unknown type from cache", zap.Any("obj", obj), zap.Any("type", reflect.TypeOf(obj))) + return nil + } +} + func (e *Enricher) Write(ev *v1.Event) { e.inputRing.Write(ev) } diff --git a/pkg/plugin/windows/hnsstats/hnsstats_windows.go b/pkg/plugin/windows/hnsstats/hnsstats_windows.go index e309bfc706..d1276c5cb0 100644 --- a/pkg/plugin/windows/hnsstats/hnsstats_windows.go +++ b/pkg/plugin/windows/hnsstats/hnsstats_windows.go @@ -13,13 +13,20 @@ import ( "github.com/Microsoft/hcsshim/hcn" v1 "github.com/cilium/cilium/pkg/hubble/api/v1" kcfg "github.com/microsoft/retina/pkg/config" + "github.com/microsoft/retina/pkg/enricher" + "github.com/microsoft/retina/pkg/exporter" "github.com/microsoft/retina/pkg/log" "github.com/microsoft/retina/pkg/metrics" "github.com/microsoft/retina/pkg/plugin/api" "github.com/microsoft/retina/pkg/utils" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) +var ( + AdvWindowsGauge *prometheus.GaugeVec +) + const ( initialize = iota + 1 start @@ -74,6 +81,11 @@ func (h *hnsstats) Init() error { } h.endpointQuery.Filter = string(filter) + if h.cfg.EnablePodLevel { + h.l.Info("Creating advanced HNS stats metrics") + initializeAdvMetrics() + } + h.l.Info("Exiting hnsstats Init...") return nil } @@ -133,6 +145,7 @@ func pullHnsStats(ctx context.Context, h *hnsstats) error { if vfpcounters, err := parseVfpPortCounters(countersRaw); err == nil { // Attach VFP port counters hnsStatsData.vfpCounters = vfpcounters + hnsStatsData.Port = portguid h.l.Debug("Attached VFP port counters", zap.String(zapPortField, portguid)) // h.l.Info(vfpcounters.String()) } else { @@ -143,6 +156,7 @@ func pullHnsStats(ctx context.Context, h *hnsstats) error { } notifyHnsStats(h, hnsStatsData) + getAdvancedMetricLabels(h, hnsStatsData) } } } @@ -197,6 +211,21 @@ func notifyHnsStats(h *hnsstats, stats *HnsStatsData) { metrics.TCPFlagGauge.WithLabelValues(egressLabel, utils.RST).Set(float64(stats.vfpCounters.Out.TcpCounters.PacketCounters.RstPacketCount)) } +func getAdvancedMetricLabels(h *hnsstats, stats *HnsStatsData) { + if AdvWindowsGauge == nil { + h.l.Warn("Advanced windows metric is not initialized") + return + } + // if port is populated, vfp data exists + labels := enricher.Instance().GetWindowLabels(stats.IPAddress) + + if labels != nil { + AdvWindowsGauge.WithLabelValues(PacketsReceived, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName, labels.Workloads.Kind, labels.Workloads.Name).Set(float64(stats.hnscounters.PacketsReceived)) + AdvWindowsGauge.WithLabelValues(PacketsSent, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName, labels.Workloads.Kind, labels.Workloads.Name).Set(float64(stats.hnscounters.PacketsSent)) + h.l.Info("updating advanced HNS stats metric", zap.String(PodName, labels.PodName), zap.String(Namespace, labels.Namespace)) + } +} + func (h *hnsstats) Start(ctx context.Context) error { h.l.Info("Start hnsstats plugin...") h.state = start @@ -223,3 +252,25 @@ func New(cfg *kcfg.Config) api.Plugin { l: log.Logger().Named(string(Name)), } } + +func cleanAdvMetrics() { + exporter.UnregisterMetric(exporter.AdvancedRegistry, metrics.ToPrometheusType(AdvWindowsGauge)) +} + +func initializeAdvMetrics() { + if AdvWindowsGauge != nil { + cleanAdvMetrics() + } + AdvWindowsGauge = exporter.CreatePrometheusGaugeVecForMetric( + exporter.AdvancedRegistry, + AdvHNSStatsName, + AdvHNSStatsDescription, + utils.Direction, + Ip, + Port, + Namespace, + PodName, + WorkloadKind, + WorkloadName, + ) +} diff --git a/pkg/plugin/windows/hnsstats/types_windows.go b/pkg/plugin/windows/hnsstats/types_windows.go index f32dbf529f..b303ef0b7e 100644 --- a/pkg/plugin/windows/hnsstats/types_windows.go +++ b/pkg/plugin/windows/hnsstats/types_windows.go @@ -10,6 +10,7 @@ import ( "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" kcfg "github.com/microsoft/retina/pkg/config" + "github.com/microsoft/retina/pkg/enricher" "github.com/microsoft/retina/pkg/log" "github.com/microsoft/retina/pkg/plugin/api" "go.opentelemetry.io/otel/attribute" @@ -21,6 +22,19 @@ import ( const ( Name api.PluginName = "hnsstats" HnsStatsEvent string = "hnsstatscount" + + // Advanced metric + AdvHNSStatsName string = "adv_windows_hns_stats" + AdvHNSStatsDescription string = "Include many different metrics from packets sent/received to closed connections" + + // Advanced metric labels + Ip string = "ip" + Port string = "port" + Namespace string = "namespace" + PodName string = "podname" + WorkloadKind string = "workload_kind" + WorkloadName string = "workload_name" + // From HNSStats API PacketsReceived string = "win_packets_recv_count" PacketsSent string = "win_packets_sent_count" @@ -74,12 +88,14 @@ type hnsstats struct { state int l *log.ZapLogger endpointQuery hcn.HostComputeQuery + enricher enricher.EnricherInterface } type HnsStatsData struct { hnscounters *hcsshim.HNSEndpointStats IPAddress string vfpCounters *VfpPortStatsData + Port string } // handles event signals such as incrementing a metric counter diff --git a/pkg/utils/attr_utils.go b/pkg/utils/attr_utils.go index 9d102dc9d7..9eefa59cdf 100644 --- a/pkg/utils/attr_utils.go +++ b/pkg/utils/attr_utils.go @@ -3,6 +3,7 @@ package utils import ( + "github.com/cilium/cilium/api/v1/flow" "go.opentelemetry.io/otel/attribute" ) @@ -86,6 +87,12 @@ var ( DNSResponseLabels = []string{"return_code", "query_type", "query", "response", "num_response"} ) +type LabelsInfo struct { + Namespace string + PodName string + Workloads *flow.Workload +} + func GetPluginEventAttributes(attrs []attribute.KeyValue, pluginName, eventName, timestamp string) []attribute.KeyValue { return append(attrs, pluginKey.String(pluginName), From b56c7ffefe431851ec7de78f76315acd72c1e8dc Mon Sep 17 00:00:00 2001 From: beegiik Date: Wed, 20 Nov 2024 17:00:57 +0000 Subject: [PATCH 2/3] WIP --- pkg/enricher/enricher.go | 2 +- pkg/plugin/windows/hnsstats/hnsstats_windows.go | 7 +++++-- pkg/utils/attr_utils.go | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/enricher/enricher.go b/pkg/enricher/enricher.go index 7155bce253..d3359b6cd3 100644 --- a/pkg/enricher/enricher.go +++ b/pkg/enricher/enricher.go @@ -202,7 +202,7 @@ func (e *Enricher) GetWindowLabels(ip string) *utils.LabelsInfo { return &utils.LabelsInfo{ Namespace: o.Namespace(), PodName: o.Name(), - Workloads: workload, + Workload: workload, } default: diff --git a/pkg/plugin/windows/hnsstats/hnsstats_windows.go b/pkg/plugin/windows/hnsstats/hnsstats_windows.go index d1276c5cb0..300b10678e 100644 --- a/pkg/plugin/windows/hnsstats/hnsstats_windows.go +++ b/pkg/plugin/windows/hnsstats/hnsstats_windows.go @@ -7,6 +7,7 @@ package hnsstats import ( "context" "encoding/json" + "fmt" "time" "github.com/Microsoft/hcsshim" @@ -220,9 +221,11 @@ func getAdvancedMetricLabels(h *hnsstats, stats *HnsStatsData) { labels := enricher.Instance().GetWindowLabels(stats.IPAddress) if labels != nil { - AdvWindowsGauge.WithLabelValues(PacketsReceived, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName, labels.Workloads.Kind, labels.Workloads.Name).Set(float64(stats.hnscounters.PacketsReceived)) - AdvWindowsGauge.WithLabelValues(PacketsSent, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName, labels.Workloads.Kind, labels.Workloads.Name).Set(float64(stats.hnscounters.PacketsSent)) + AdvWindowsGauge.WithLabelValues(PacketsReceived, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName, labels.Workload.Kind, labels.Workload.Name).Set(float64(stats.hnscounters.PacketsReceived)) + AdvWindowsGauge.WithLabelValues(PacketsSent, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName, labels.Workload.Kind, labels.Workload.Name).Set(float64(stats.hnscounters.PacketsSent)) h.l.Info("updating advanced HNS stats metric", zap.String(PodName, labels.PodName), zap.String(Namespace, labels.Namespace)) + fmt.Printf("\n%+v", labels.Workload.Kind) + fmt.Printf("\n%+v", labels.Workload.Name) } } diff --git a/pkg/utils/attr_utils.go b/pkg/utils/attr_utils.go index 9eefa59cdf..37d9d70961 100644 --- a/pkg/utils/attr_utils.go +++ b/pkg/utils/attr_utils.go @@ -90,7 +90,7 @@ var ( type LabelsInfo struct { Namespace string PodName string - Workloads *flow.Workload + Workload *flow.Workload } func GetPluginEventAttributes(attrs []attribute.KeyValue, pluginName, eventName, timestamp string) []attribute.KeyValue { From 8a41b1e441e3933c8e08b9a181b0b2a9a036ba5a Mon Sep 17 00:00:00 2001 From: beegiik Date: Wed, 20 Nov 2024 17:15:36 +0000 Subject: [PATCH 3/3] WIP --- pkg/enricher/enricher.go | 2 ++ pkg/plugin/windows/hnsstats/hnsstats_windows.go | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/enricher/enricher.go b/pkg/enricher/enricher.go index d3359b6cd3..2f6995bbbb 100644 --- a/pkg/enricher/enricher.go +++ b/pkg/enricher/enricher.go @@ -5,6 +5,7 @@ package enricher import ( "context" + "fmt" "reflect" "sync" @@ -194,6 +195,7 @@ func (e *Enricher) GetWindowLabels(ip string) *utils.LabelsInfo { case *common.RetinaEndpoint: var workload *flow.Workload if workloads := e.getWorkloads(o.OwnerRefs()); len(workloads) > 0 { + fmt.Printf("Workloads Check: \n%+v", workloads) workload = workloads[0] } else { workload = &flow.Workload{Name: "", Kind: ""} diff --git a/pkg/plugin/windows/hnsstats/hnsstats_windows.go b/pkg/plugin/windows/hnsstats/hnsstats_windows.go index 300b10678e..b8faafbab7 100644 --- a/pkg/plugin/windows/hnsstats/hnsstats_windows.go +++ b/pkg/plugin/windows/hnsstats/hnsstats_windows.go @@ -7,7 +7,6 @@ package hnsstats import ( "context" "encoding/json" - "fmt" "time" "github.com/Microsoft/hcsshim" @@ -223,9 +222,7 @@ func getAdvancedMetricLabels(h *hnsstats, stats *HnsStatsData) { if labels != nil { AdvWindowsGauge.WithLabelValues(PacketsReceived, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName, labels.Workload.Kind, labels.Workload.Name).Set(float64(stats.hnscounters.PacketsReceived)) AdvWindowsGauge.WithLabelValues(PacketsSent, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName, labels.Workload.Kind, labels.Workload.Name).Set(float64(stats.hnscounters.PacketsSent)) - h.l.Info("updating advanced HNS stats metric", zap.String(PodName, labels.PodName), zap.String(Namespace, labels.Namespace)) - fmt.Printf("\n%+v", labels.Workload.Kind) - fmt.Printf("\n%+v", labels.Workload.Name) + h.l.Info("updating advanced HNS stats metric", zap.String(PodName, labels.PodName), zap.String(Namespace, labels.Namespace), zap.String(labels.Workload.Kind, labels.Workload.Name)) } }