diff --git a/cmd/legacy/daemon.go b/cmd/legacy/daemon.go index 6e06ee73cc..fe6e17a07d 100644 --- a/cmd/legacy/daemon.go +++ b/cmd/legacy/daemon.go @@ -89,13 +89,16 @@ func (d *Daemon) Start() error { } daemonConfig, err := config.GetConfig(d.configFile) + fmt.Printf("config %+v\n", daemonConfig) + if err != nil { panic(err) } fmt.Println("init client-go") var cfg *rest.Config - if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { + var kubeconfig = os.Getenv("KUBECONFIG") + if kubeconfig != "" { fmt.Println("KUBECONFIG set, using kubeconfig: ", kubeconfig) cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { @@ -104,7 +107,8 @@ func (d *Daemon) Start() error { } else { cfg, err = kcfg.GetConfig() if err != nil { - panic(err) + fmt.Println("KUBECONFIG not set. Falling back to standalone mode") + cfg = loadStandaloneConfig() } } @@ -208,6 +212,11 @@ func (d *Daemon) Start() error { } } + // Setup RetinaEndpoint controller. + // TODO(mainred): This is to temporarily create a cache and pubsub for RetinaEndpoint, need to refactor this. + ctx := ctrl.SetupSignalHandler() + ctrl.SetLogger(zapr.NewLogger(zl.Logger.Named("controller-runtime"))) + mgr, err := crmgr.New(cfg, mgrOption) if err != nil { mainLogger.Error("Unable to start manager", zap.Error(err)) @@ -226,18 +235,15 @@ func (d *Daemon) Start() error { // k8s Client used for informers cl := kubernetes.NewForConfigOrDie(mgr.GetConfig()) - serverVersion, err := cl.Discovery().ServerVersion() - if err != nil { - mainLogger.Error("failed to get Kubernetes server version: ", zap.Error(err)) - } else { - mainLogger.Infof("Kubernetes server version: %v", serverVersion) + if kubeconfig != "" { + serverVersion, err := cl.Discovery().ServerVersion() + if err != nil { + mainLogger.Error("failed to get Kubernetes server version: ", zap.Error(err)) + } else { + mainLogger.Infof("Kubernetes server version: %v", serverVersion) + } } - // Setup RetinaEndpoint controller. - // TODO(mainred): This is to temporarily create a cache and pubsub for RetinaEndpoint, need to refactor this. - ctx := ctrl.SetupSignalHandler() - ctrl.SetLogger(zapr.NewLogger(zl.Logger.Named("controller-runtime"))) - if daemonConfig.EnablePodLevel { pubSub := pubsub.New() controllerCache := controllercache.New(pubSub) @@ -251,47 +257,49 @@ func (d *Daemon) Start() error { enrich.Run() metricsModule := mm.InitModule(ctx, daemonConfig, pubSub, enrich, fm, controllerCache) - if !daemonConfig.RemoteContext { - mainLogger.Info("Initializing Pod controller") - - podController := pc.New(mgr.GetClient(), controllerCache) - if err := podController.SetupWithManager(mgr); err != nil { - mainLogger.Fatal("unable to create PodController", zap.Error(err)) + if cfg != nil { + if !daemonConfig.RemoteContext { + mainLogger.Info("Initializing Pod controller") + + podController := pc.New(mgr.GetClient(), controllerCache) + if err := podController.SetupWithManager(mgr); err != nil { + mainLogger.Fatal("unable to create PodController", zap.Error(err)) + } + } else if daemonConfig.EnableRetinaEndpoint { + mainLogger.Info("RetinaEndpoint is enabled") + mainLogger.Info("Initializing RetinaEndpoint controller") + + retinaEndpointController := kec.New(mgr.GetClient(), controllerCache) + if err := retinaEndpointController.SetupWithManager(mgr); err != nil { + mainLogger.Fatal("unable to create retinaEndpointController", zap.Error(err)) + } } - } else if daemonConfig.EnableRetinaEndpoint { - mainLogger.Info("RetinaEndpoint is enabled") - mainLogger.Info("Initializing RetinaEndpoint controller") - retinaEndpointController := kec.New(mgr.GetClient(), controllerCache) - if err := retinaEndpointController.SetupWithManager(mgr); err != nil { - mainLogger.Fatal("unable to create retinaEndpointController", zap.Error(err)) + mainLogger.Info("Initializing Node controller") + nodeController := nc.New(mgr.GetClient(), controllerCache) + if err := nodeController.SetupWithManager(mgr); err != nil { + mainLogger.Fatal("unable to create nodeController", zap.Error(err)) } - } - - mainLogger.Info("Initializing Node controller") - nodeController := nc.New(mgr.GetClient(), controllerCache) - if err := nodeController.SetupWithManager(mgr); err != nil { - mainLogger.Fatal("unable to create nodeController", zap.Error(err)) - } - - mainLogger.Info("Initializing Service controller") - svcController := sc.New(mgr.GetClient(), controllerCache) - if err := svcController.SetupWithManager(mgr); err != nil { - mainLogger.Fatal("unable to create svcController", zap.Error(err)) - } - if daemonConfig.EnableAnnotations { - mainLogger.Info("Initializing MetricsConfig namespaceController") - namespaceController := namespacecontroller.New(mgr.GetClient(), controllerCache, metricsModule) - if err := namespaceController.SetupWithManager(mgr); err != nil { - mainLogger.Fatal("unable to create namespaceController", zap.Error(err)) + mainLogger.Info("Initializing Service controller") + svcController := sc.New(mgr.GetClient(), controllerCache) + if err := svcController.SetupWithManager(mgr); err != nil { + mainLogger.Fatal("unable to create svcController", zap.Error(err)) } - go namespaceController.Start(ctx) - } else { - mainLogger.Info("Initializing MetricsConfig controller") - metricsConfigController := mcc.New(mgr.GetClient(), mgr.GetScheme(), metricsModule) - if err := metricsConfigController.SetupWithManager(mgr); err != nil { - mainLogger.Fatal("unable to create metricsConfigController", zap.Error(err)) + + if daemonConfig.EnableAnnotations { + mainLogger.Info("Initializing MetricsConfig namespaceController") + namespaceController := namespacecontroller.New(mgr.GetClient(), controllerCache, metricsModule) + if err := namespaceController.SetupWithManager(mgr); err != nil { + mainLogger.Fatal("unable to create namespaceController", zap.Error(err)) + } + go namespaceController.Start(ctx) + } else { + mainLogger.Info("Initializing MetricsConfig controller") + metricsConfigController := mcc.New(mgr.GetClient(), mgr.GetScheme(), metricsModule) + if err := metricsConfigController.SetupWithManager(mgr); err != nil { + mainLogger.Fatal("unable to create metricsConfigController", zap.Error(err)) + } } } } @@ -323,3 +331,14 @@ func (d *Daemon) Start() error { mainLogger.Info("Network observability exiting. Till next time!") return nil } + +func loadStandaloneConfig() *rest.Config { + return &rest.Config{ + Host: "http://localhost:8080", + APIPath: "/api", + ContentConfig: rest.ContentConfig{ + GroupVersion: nil, + NegotiatedSerializer: nil, + }, + } +} diff --git a/pkg/enricher/enricher.go b/pkg/enricher/enricher.go index 98013cd2ca..86b193b3bd 100644 --- a/pkg/enricher/enricher.go +++ b/pkg/enricher/enricher.go @@ -14,6 +14,7 @@ import ( "github.com/microsoft/retina/pkg/common" "github.com/microsoft/retina/pkg/controllers/cache" "github.com/microsoft/retina/pkg/log" + "github.com/microsoft/retina/pkg/utils" "go.uber.org/zap" ) @@ -186,6 +187,26 @@ func (e *Enricher) Write(ev *v1.Event) { e.inputRing.Write(ev) } +func (e *Enricher) GetWindowLabels(ip string) *utils.LabelsInfo { + obj := e.cache.GetObjByIP(ip) + if obj == nil { + e.l.Debug("No object found for IP", zap.String("ip", ip)) + return nil + } + + switch o := obj.(type) { + case *common.RetinaEndpoint: + return &utils.LabelsInfo{ + Namespace: o.Namespace(), + PodName: o.Name(), + } + + default: + e.l.Debug("received unknown type from cache", zap.Any("obj", obj), zap.Any("type", reflect.TypeOf(obj))) + return nil + } +} + func (e *Enricher) ExportReader() *container.RingReader { return container.NewRingReader(e.outputRing, e.outputRing.OldestWrite()) } diff --git a/pkg/hubble/common/mocks/mock_types.go b/pkg/hubble/common/mocks/mock_types.go new file mode 100644 index 0000000000..aa2b561a19 --- /dev/null +++ b/pkg/hubble/common/mocks/mock_types.go @@ -0,0 +1,5 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: decoder.go + +// Package mocks is a generated GoMock package. +package mocks diff --git a/pkg/managers/pluginmanager/pluginmanager.go b/pkg/managers/pluginmanager/pluginmanager.go index 15d133382f..3e952e4344 100644 --- a/pkg/managers/pluginmanager/pluginmanager.go +++ b/pkg/managers/pluginmanager/pluginmanager.go @@ -59,6 +59,8 @@ func NewPluginManager(cfg *kcfg.Config, tel telemetry.Telemetry) (*PluginManager } for _, name := range cfg.EnabledPlugin { + fmt.Printf("Enabled plugins : %+v\n", name) + newPluginFn, ok := plugin.Get(name) if !ok { return nil, fmt.Errorf("plugin %s not found in registry", name) diff --git a/pkg/plugin/conntrack/conntrack_bpfel_x86.o b/pkg/plugin/conntrack/conntrack_bpfel_x86.o index e69de29bb2..b2adc2bab7 100644 Binary files a/pkg/plugin/conntrack/conntrack_bpfel_x86.o and b/pkg/plugin/conntrack/conntrack_bpfel_x86.o differ diff --git a/pkg/plugin/dropreason/kprobe_bpfel_x86.o b/pkg/plugin/dropreason/kprobe_bpfel_x86.o index e69de29bb2..a0098ede38 100644 Binary files a/pkg/plugin/dropreason/kprobe_bpfel_x86.o and b/pkg/plugin/dropreason/kprobe_bpfel_x86.o differ diff --git a/pkg/plugin/filter/filter_bpfel_x86.o b/pkg/plugin/filter/filter_bpfel_x86.o index e69de29bb2..bf6c879c81 100644 Binary files a/pkg/plugin/filter/filter_bpfel_x86.o and b/pkg/plugin/filter/filter_bpfel_x86.o differ diff --git a/pkg/plugin/hnsstats/hnsstats_windows.go b/pkg/plugin/hnsstats/hnsstats_windows.go index a668dce4f9..a60309f992 100644 --- a/pkg/plugin/hnsstats/hnsstats_windows.go +++ b/pkg/plugin/hnsstats/hnsstats_windows.go @@ -7,19 +7,27 @@ package hnsstats import ( "context" "encoding/json" + "fmt" + "os" "time" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" v1 "github.com/cilium/cilium/pkg/hubble/api/v1" kcfg "github.com/microsoft/retina/pkg/config" + "github.com/microsoft/retina/pkg/exporter" "github.com/microsoft/retina/pkg/log" "github.com/microsoft/retina/pkg/metrics" "github.com/microsoft/retina/pkg/plugin/registry" "github.com/microsoft/retina/pkg/utils" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) +var ( + AdvWindowsGauge *prometheus.GaugeVec +) + const ( initialize = iota + 1 start @@ -85,6 +93,11 @@ func (h *hnsstats) Init() error { } h.endpointQuery.Filter = string(filter) + // if h.cfg.EnablePodLevel { + h.l.Info("Creating advanced HNS stats metrics") + initializeAdvMetrics() + // } + h.l.Info("Exiting hnsstats Init...") return nil } @@ -144,6 +157,7 @@ func pullHnsStats(ctx context.Context, h *hnsstats) error { if vfpcounters, err := parseVfpPortCounters(countersRaw); err == nil { // Attach VFP port counters hnsStatsData.vfpCounters = vfpcounters + hnsStatsData.Port = portguid h.l.Debug("Attached VFP port counters", zap.String(zapPortField, portguid)) // h.l.Info(vfpcounters.String()) } else { @@ -154,6 +168,7 @@ func pullHnsStats(ctx context.Context, h *hnsstats) error { } notifyHnsStats(h, hnsStatsData) + getAdvancedMetricLabels(h, hnsStatsData) } } } @@ -208,12 +223,114 @@ func notifyHnsStats(h *hnsstats, stats *HnsStatsData) { metrics.TCPFlagGauge.WithLabelValues(egressLabel, utils.RST).Set(float64(stats.vfpCounters.Out.TcpCounters.PacketCounters.RstPacketCount)) } +func getAdvancedMetricLabels(h *hnsstats, stats *HnsStatsData) { + if AdvWindowsGauge == nil { + h.l.Warn("Advanced windows metric is not initialized") + return + } + // if port is populated, vfp data exists + // labels := enricher.Instance().GetWindowLabels(stats.IPAddress) + h.l.Info("New standalone feature") + + labels, err := getStandaloneLabels(stats.IPAddress) + h.l.Info("Reading fields from CNI state file", zap.String(Ip, stats.IPAddress), zap.String(PodName, labels.PodName), zap.String(Namespace, labels.Namespace)) + + if err != nil { + AdvWindowsGauge.WithLabelValues(PacketsReceived, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName).Set(float64(stats.hnscounters.PacketsReceived)) + AdvWindowsGauge.WithLabelValues(PacketsSent, stats.IPAddress, stats.Port, labels.Namespace, labels.PodName).Set(float64(stats.hnscounters.PacketsSent)) + h.l.Info("updating advanced HNS stats metric", zap.String(PodName, labels.PodName), zap.String(Namespace, labels.Namespace)) + } +} + func (h *hnsstats) Start(ctx context.Context) error { h.l.Info("Start hnsstats plugin...") h.state = start return pullHnsStats(ctx, h) } +func cleanAdvMetrics() { + exporter.UnregisterMetric(exporter.AdvancedRegistry, metrics.ToPrometheusType(AdvWindowsGauge)) +} + +func initializeAdvMetrics() { + if AdvWindowsGauge != nil { + cleanAdvMetrics() + } + AdvWindowsGauge = exporter.CreatePrometheusGaugeVecForMetric( + exporter.AdvancedRegistry, + AdvHNSStatsName, + AdvHNSStatsDescription, + utils.Direction, + Ip, + Port, + Namespace, + PodName, + ) +} + +type Endpoint struct { + ID string `json:"Id"` + IPAddresses []IPInfo `json:"IPAddresses"` + PODName string `json:"PODName"` + PODNameSpace string `json:"PODNameSpace"` +} + +type IPInfo struct { + IP string `json:"IP"` +} + +type Network struct { + ExternalInterfaces map[string]ExternalInterface `json:"ExternalInterfaces"` +} + +type ExternalInterface struct { + Networks map[string]NetworkInfo `json:"Networks"` +} + +type NetworkInfo struct { + Endpoints map[string]Endpoint `json:"Endpoints"` +} + +func getStandaloneLabels(ip string) (*utils.LabelsInfo, error) { + file, err := os.Open("C:/k/azure-vnet.json") + if err != nil { + return &utils.LabelsInfo{ + Namespace: "", + PodName: "", + }, fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + + var network Network + if err := json.NewDecoder(file).Decode(&network); err != nil { + return &utils.LabelsInfo{ + Namespace: "", + PodName: "", + }, fmt.Errorf("failed to decode JSON: %w", err) + } + + // Search for the IP in Endpoints + for _, iface := range network.ExternalInterfaces { + for _, network := range iface.Networks { + for _, endpoint := range network.Endpoints { + for _, ipAddr := range endpoint.IPAddresses { + if ipAddr.IP == ip { + return &utils.LabelsInfo{ + Namespace: endpoint.PODNameSpace, + PodName: endpoint.PODName, + }, nil + } + } + } + } + } + + return &utils.LabelsInfo{ + Namespace: "", + PodName: "", + }, fmt.Errorf("IP address not found") +} + func (d *hnsstats) Stop() error { d.l.Info("Entered hnsstats Stop...") if d.state != start { diff --git a/pkg/plugin/hnsstats/types_windows.go b/pkg/plugin/hnsstats/types_windows.go index ab8511d5d3..5eed46d257 100644 --- a/pkg/plugin/hnsstats/types_windows.go +++ b/pkg/plugin/hnsstats/types_windows.go @@ -10,6 +10,7 @@ import ( "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" kcfg "github.com/microsoft/retina/pkg/config" + "github.com/microsoft/retina/pkg/enricher" "github.com/microsoft/retina/pkg/log" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" @@ -20,6 +21,19 @@ import ( const ( name string = "hnsstats" HnsStatsEvent string = "hnsstatscount" + + // Advanced metric + AdvHNSStatsName string = "adv_windows_hns_stats" + AdvHNSStatsDescription string = "Include many different metrics from packets sent/received to closed connections" + + // Advanced metric labels + Ip string = "ip" + Port string = "port" + Namespace string = "namespace" + PodName string = "podname" + WorkloadKind string = "workload_kind" + WorkloadName string = "workload_name" + // From HNSStats API PacketsReceived string = "win_packets_recv_count" PacketsSent string = "win_packets_sent_count" @@ -73,12 +87,14 @@ type hnsstats struct { state int l *log.ZapLogger endpointQuery hcn.HostComputeQuery + enricher enricher.EnricherInterface } type HnsStatsData struct { hnscounters *hcsshim.HNSEndpointStats IPAddress string vfpCounters *VfpPortStatsData + Port string } // handles event signals such as incrementing a metric counter diff --git a/pkg/plugin/mock/plugin.go b/pkg/plugin/mock/plugin.go index 888d9267dd..403a472c3c 100644 --- a/pkg/plugin/mock/plugin.go +++ b/pkg/plugin/mock/plugin.go @@ -5,11 +5,11 @@ // // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/microsoft/retina/pkg/plugin/ (interfaces: Plugin) +// Source: github.com/microsoft/retina/pkg/plugin (interfaces: Plugin) // // Generated by this command: // -// mockgen -destination=mock/plugin.go -copyright_file=../lib/ignore_headers.txt -package=plugin github.com/microsoft/retina/pkg/plugin/ Plugin +// mockgen -destination=mock/plugin.go -copyright_file=../lib/ignore_headers.txt -package=plugin github.com/microsoft/retina/pkg/plugin Plugin // // Package plugin is a generated GoMock package. @@ -27,7 +27,6 @@ import ( type MockPlugin struct { ctrl *gomock.Controller recorder *MockPluginMockRecorder - isgomock struct{} } // MockPluginMockRecorder is the mock recorder for MockPlugin. @@ -48,31 +47,31 @@ func (m *MockPlugin) EXPECT() *MockPluginMockRecorder { } // Compile mocks base method. -func (m *MockPlugin) Compile(ctx context.Context) error { +func (m *MockPlugin) Compile(arg0 context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Compile", ctx) + ret := m.ctrl.Call(m, "Compile", arg0) ret0, _ := ret[0].(error) return ret0 } // Compile indicates an expected call of Compile. -func (mr *MockPluginMockRecorder) Compile(ctx any) *gomock.Call { +func (mr *MockPluginMockRecorder) Compile(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compile", reflect.TypeOf((*MockPlugin)(nil).Compile), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compile", reflect.TypeOf((*MockPlugin)(nil).Compile), arg0) } // Generate mocks base method. -func (m *MockPlugin) Generate(ctx context.Context) error { +func (m *MockPlugin) Generate(arg0 context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Generate", ctx) + ret := m.ctrl.Call(m, "Generate", arg0) ret0, _ := ret[0].(error) return ret0 } // Generate indicates an expected call of Generate. -func (mr *MockPluginMockRecorder) Generate(ctx any) *gomock.Call { +func (mr *MockPluginMockRecorder) Generate(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Generate", reflect.TypeOf((*MockPlugin)(nil).Generate), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Generate", reflect.TypeOf((*MockPlugin)(nil).Generate), arg0) } // Init mocks base method. @@ -118,17 +117,17 @@ func (mr *MockPluginMockRecorder) SetupChannel(arg0 any) *gomock.Call { } // Start mocks base method. -func (m *MockPlugin) Start(ctx context.Context) error { +func (m *MockPlugin) Start(arg0 context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Start", ctx) + ret := m.ctrl.Call(m, "Start", arg0) ret0, _ := ret[0].(error) return ret0 } // Start indicates an expected call of Start. -func (mr *MockPluginMockRecorder) Start(ctx any) *gomock.Call { +func (mr *MockPluginMockRecorder) Start(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockPlugin)(nil).Start), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockPlugin)(nil).Start), arg0) } // Stop mocks base method. diff --git a/pkg/plugin/packetforward/packetforward_bpfel_x86.o b/pkg/plugin/packetforward/packetforward_bpfel_x86.o index e69de29bb2..5f17550026 100644 Binary files a/pkg/plugin/packetforward/packetforward_bpfel_x86.o and b/pkg/plugin/packetforward/packetforward_bpfel_x86.o differ diff --git a/pkg/plugin/packetparser/packetparser_bpfel_x86.o b/pkg/plugin/packetparser/packetparser_bpfel_x86.o index e69de29bb2..dddcc7dd9b 100644 Binary files a/pkg/plugin/packetparser/packetparser_bpfel_x86.o and b/pkg/plugin/packetparser/packetparser_bpfel_x86.o differ diff --git a/pkg/utils/attr_utils.go b/pkg/utils/attr_utils.go index 6826db74c9..4bac0a0d7c 100644 --- a/pkg/utils/attr_utils.go +++ b/pkg/utils/attr_utils.go @@ -87,6 +87,11 @@ var ( DNSResponseLabels = []string{"return_code", "query_type", "query", "response", "num_response"} ) +type LabelsInfo struct { + Namespace string + PodName string +} + func GetPluginEventAttributes(attrs []attribute.KeyValue, pluginName, eventName, timestamp string) []attribute.KeyValue { return append(attrs, pluginKey.String(pluginName), diff --git a/winpod.yaml b/winpod.yaml new file mode 100644 index 0000000000..1234d3db78 --- /dev/null +++ b/winpod.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: privileged-daemonset + namespace: kube-system + labels: + app: privileged-daemonset +spec: + selector: + matchLabels: + app: privileged-daemonset + template: + metadata: + labels: + app: privileged-daemonset + spec: + nodeSelector: + kubernetes.io/os: windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: powershell + image: mcr.microsoft.com/powershell:lts-nanoserver-1809 # or lts-nanoserver-ltsc2022 + command: + - powershell.exe + - -Command + - Start-Sleep -Seconds 2147483 + terminationGracePeriodSeconds: 0