Merge branch 'microsoft:main' into amilcara/updating-grafana-dashboards

microsoft · Feb 5, 2025 · 140b73f · 140b73f
2 parents fb38af2 + a1beefd
commit 140b73f
Show file tree

Hide file tree

Showing 72 changed files with 1,570 additions and 325 deletions.
diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml
@@ -52,6 +52,7 @@ jobs:
               IMAGE_NAMESPACE=${{ github.repository }} \
               PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \
               IMAGE_REGISTRY=${{ vars.ACR_NAME }}  \
+              APP_INSIGHTS_ID=${{ secrets.AZURE_APP_INSIGHTS_ID }} \
               BUILDX_ACTION=--push
           else
             make retina-image \
@@ -102,6 +103,7 @@ jobs:
               IMAGE_NAMESPACE=${{ github.repository }} \
               PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \
               IMAGE_REGISTRY=${{ vars.ACR_NAME }}  \
+              APP_INSIGHTS_ID=${{ secrets.AZURE_APP_INSIGHTS_ID }} \
               WINDOWS_YEARS=${{ matrix.year }} \
               BUILDX_ACTION=--push
           else
@@ -153,6 +155,7 @@ jobs:
               IMAGE_NAMESPACE=${{ github.repository }} \
               PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \
               IMAGE_REGISTRY=${{ vars.ACR_NAME }} \
+              APP_INSIGHTS_ID=${{ secrets.AZURE_APP_INSIGHTS_ID }} \
               BUILDX_ACTION=--push
           else
             make retina-operator-image \
@@ -332,9 +335,9 @@ jobs:
           set -euo pipefail
           go test -v ./test/e2e/. -timeout 60m -tags=e2e -count=1  -args -image-tag=$(make version) -image-registry=${{ vars.ACR_NAME }}  -image-namespace=${{ github.repository}}
 
-  perf:
+  perf-basic:
     if: ${{ github.event_name == 'merge_group' && success('manifests')}}
-    name: Retina Performance Test
+    name: Retina Basic Performance Test
     runs-on: ubuntu-latest
     needs: [manifests]
 
@@ -345,7 +348,41 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v5
         with:
-          go-version-file: go.mod
+          go-version: 1.23
+      - run: go version
+
+      - name: Az CLI login
+        uses: azure/login@v2
+        if: ${{ github.event_name == 'merge_group' }}
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION }}
+
+      - name: Run Perf Tests
+        env:
+          AZURE_APP_INSIGHTS_KEY: ${{ secrets.AZURE_APP_INSIGHTS_KEY }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION }}
+          AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          go test -v ./test/e2e/. -timeout 2h -tags=perf -count=1 -args -image-tag=$(make version) -image-registry=${{ vars.ACR_NAME }} -image-namespace=${{ github.repository }} -retina-mode=basic
+
+  perf-advanced:
+    if: ${{ github.event_name == 'merge_group' && success('manifests')}}
+    name: Retina Advanced Performance Test
+    runs-on: ubuntu-latest
+    needs: [manifests]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.23
       - run: go version
 
       - name: Az CLI login
@@ -364,4 +401,4 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          go test -v ./test/e2e/. -timeout 2h -tags=perf -count=1 -args -image-tag=$(make version) -image-registry=${{ vars.ACR_NAME }} -image-namespace=${{ github.repository }}
+          go test -v ./test/e2e/. -timeout 2h -tags=perf -count=1 -args -image-tag=$(make version) -image-registry=${{ vars.ACR_NAME }} -image-namespace=${{ github.repository }} -retina-mode=advanced
diff --git a/.github/workflows/perf.yaml b/.github/workflows/perf.yaml
@@ -15,8 +15,8 @@ permissions:
   id-token: write
 
 jobs:
-  perf_test:
-    name: Retina Performance Test
+  perf-test-basic:
+    name: Retina Basic Performance Test
     runs-on: ubuntu-latest
 
     steps:
@@ -26,7 +26,7 @@ jobs:
       - name: Setup go
         uses: actions/setup-go@v5
         with:
-          go-version-file: go.mod
+          go-version: 1.23
       - run: go version
 
       - name: Az CLI login
@@ -49,4 +49,40 @@ jobs:
           else
             TAG=$(curl -s https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .tag_name)
           fi
-          go test -v ./test/e2e/. -timeout 2h -tags=perf -count=1 -args -image-tag=$TAG -image-registry=ghcr.io -image-namespace=${{ github.repository }}
+          go test -v ./test/e2e/. -timeout 2h -tags=perf -count=1 -args -image-tag=$TAG -image-registry=ghcr.io -image-namespace=${{ github.repository }} -retina-mode=basic
+
+  perf-test-advanced:
+    name: Retina Advanced Performance Test
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.23
+      - run: go version
+
+      - name: Az CLI login
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION }}
+
+      - name: Run performance measurement for Retina
+        env:
+          AZURE_APP_INSIGHTS_KEY: ${{ secrets.AZURE_APP_INSIGHTS_KEY }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION }}
+          AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            TAG=${{ github.event.inputs.tag }}
+          else
+            TAG=$(curl -s https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .tag_name)
+          fi
+          go test -v ./test/e2e/. -timeout 2h -tags=perf -count=1 -args -image-tag=$TAG -image-registry=ghcr.io -image-namespace=${{ github.repository }} -retina-mode=advanced
diff --git a/Makefile b/Makefile
@@ -35,7 +35,7 @@ PLATFORM		?= $(OS)/$(ARCH)
 PLATFORMS		?= linux/amd64 linux/arm64 windows/amd64
 OS_VERSION		?= ltsc2019
 
-HUBBLE_VERSION ?= v1.16.5 # This may be modified via the update-hubble GitHub Action
+HUBBLE_VERSION ?= v1.16.6 # This may be modified via the update-hubble GitHub Action
 
 CONTAINER_BUILDER ?= docker
 CONTAINER_RUNTIME ?= docker

diff --git a/cli/Dockerfile b/cli/Dockerfile
@@ -1,5 +1,5 @@
-# mcr.microsoft.com/oss/go/microsoft/golang:1.23.4-cbl-mariner2.0
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:88225e171f29fe5f1f6ffca8eb659535b19b253354e43e1f4fc8a9bc67615ca1 AS builder
+# mcr.microsoft.com/oss/go/microsoft/golang:1.23.5-cbl-mariner2.0
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:ef39e430a97a61bec3395adb673a7c5611d2212f14abe40da21cbfd11a9d6e9d AS builder
 
 
 ARG VERSION

diff --git a/cmd/standard/daemon.go b/cmd/standard/daemon.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"time"
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
@@ -48,8 +47,7 @@ import (
 )
 
 const (
-	logFileName       = "retina.log"
-	heartbeatInterval = 15 * time.Minute
+	logFileName = "retina.log"
 
 	nodeNameEnvKey = "NODE_NAME"
 	nodeIPEnvKey   = "NODE_IP"
@@ -309,7 +307,7 @@ func (d *Daemon) Start() error {
 	defer controllerMgr.Stop(ctx)
 
 	// start heartbeat goroutine for application insights
-	go tel.Heartbeat(ctx, heartbeatInterval)
+	go tel.Heartbeat(ctx, daemonConfig.TelemetryInterval)
 
 	// Start controller manager, which will start http server and plugin manager.
 	go controllerMgr.Start(ctx)

diff --git a/controller/Dockerfile b/controller/Dockerfile
@@ -2,8 +2,8 @@ ARG OS_VERSION
 
 # pinned base images
 
-# mcr.microsoft.com/oss/go/microsoft/golang:1.23.4-cbl-mariner2.0
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:88225e171f29fe5f1f6ffca8eb659535b19b253354e43e1f4fc8a9bc67615ca1 AS golang
+# mcr.microsoft.com/oss/go/microsoft/golang:1.23.5-cbl-mariner2.0
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:ef39e430a97a61bec3395adb673a7c5611d2212f14abe40da21cbfd11a9d6e9d AS golang
 
 # mcr.microsoft.com/cbl-mariner/base/core:2.0
 FROM --platform=$TARGETPLATFORM mcr.microsoft.com/cbl-mariner/base/core@sha256:77651116f2e83cf50fddd8a0316945499f8ce6521ff8e94e67539180d1e5975a AS mariner-core
@@ -99,7 +99,7 @@ RUN arr="clang tcpdump ip ss iptables-legacy iptables-legacy-save iptables-nft i
 ARG GOARCH=amd64
 ENV HUBBLE_ARCH=${GOARCH}
 # ARG HUBBLE_VERSION may be modified via the update-hubble GitHub Action
-ARG HUBBLE_VERSION=v1.16.5
+ARG HUBBLE_VERSION=v1.16.6
 ENV HUBBLE_VERSION=${HUBBLE_VERSION}
 RUN echo "Hubble version: $HUBBLE_VERSION" && \
     wget --no-check-certificate https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-${HUBBLE_ARCH}.tar.gz && \

diff --git a/controller/Dockerfile.gogen b/controller/Dockerfile.gogen
@@ -1,5 +1,5 @@
-# mcr.microsoft.com/oss/go/microsoft/golang:1.23.4-cbl-mariner2.0
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:88225e171f29fe5f1f6ffca8eb659535b19b253354e43e1f4fc8a9bc67615ca1
+# mcr.microsoft.com/oss/go/microsoft/golang:1.23.5-cbl-mariner2.0
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:ef39e430a97a61bec3395adb673a7c5611d2212f14abe40da21cbfd11a9d6e9d
 
 
 # Default linux/architecture.

diff --git a/controller/Dockerfile.proto b/controller/Dockerfile.proto
@@ -1,5 +1,5 @@
-# mcr.microsoft.com/oss/go/microsoft/golang:1.23.4-cbl-mariner2.0
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:88225e171f29fe5f1f6ffca8eb659535b19b253354e43e1f4fc8a9bc67615ca1
+# mcr.microsoft.com/oss/go/microsoft/golang:1.23.5-cbl-mariner2.0
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:ef39e430a97a61bec3395adb673a7c5611d2212f14abe40da21cbfd11a9d6e9d
 
 LABEL Name=retina-builder Version=0.0.1
 

diff --git a/controller/Dockerfile.windows-2022 b/controller/Dockerfile.windows-2022
@@ -1,5 +1,5 @@
-# mcr.microsoft.com/oss/go/microsoft/golang:1.23.4-cbl-mariner2.0
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:88225e171f29fe5f1f6ffca8eb659535b19b253354e43e1f4fc8a9bc67615ca1 AS builder
+# mcr.microsoft.com/oss/go/microsoft/golang:1.23.5-cbl-mariner2.0
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang@sha256:ef39e430a97a61bec3395adb673a7c5611d2212f14abe40da21cbfd11a9d6e9d AS builder
 
 # Build args
 ARG VERSION

diff --git a/controller/Dockerfile.windows-cgo b/controller/Dockerfile.windows-cgo
@@ -1,5 +1,5 @@
-# mcr.microsoft.com/oss/go/microsoft/golang:1.23.4-windowsservercore-ltsc2022
-FROM --platform=windows/amd64 mcr.microsoft.com/oss/go/microsoft/golang@sha256:e423369b45d4144324e61d6d64ce7a6d0e2bdf7f181c74d8ed25346891b8aacd AS cgo
+# mcr.microsoft.com/oss/go/microsoft/golang:1.23.5-windowsservercore-ltsc2022
+FROM --platform=windows/amd64 mcr.microsoft.com/oss/go/microsoft/golang@sha256:f88b5ef1afd628c18196e98c3d1c6081617768f4ad316433dd29b19a257a46eb AS cgo
 
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 

diff --git a/controller/Dockerfile.windows-native b/controller/Dockerfile.windows-native
@@ -3,8 +3,8 @@
 # buildx targets, and this one requires legacy build.
 # Maybe one day: https://github.com/moby/buildkit/issues/616
 ARG BUILDER_IMAGE
-# mcr.microsoft.com/oss/go/microsoft/golang:1.23.4-windowsservercore-ltsc2022
-FROM --platform=windows/amd64 mcr.microsoft.com/oss/go/microsoft/golang@sha256:e423369b45d4144324e61d6d64ce7a6d0e2bdf7f181c74d8ed25346891b8aacd AS builder
+# mcr.microsoft.com/oss/go/microsoft/golang:1.23.5-windowsservercore-ltsc2022
+FROM --platform=windows/amd64 mcr.microsoft.com/oss/go/microsoft/golang@sha256:f88b5ef1afd628c18196e98c3d1c6081617768f4ad316433dd29b19a257a46eb AS builder
 WORKDIR C:\\retina
 COPY go.mod .
 COPY go.sum .

diff --git a/deploy/standard/manifests/controller/helm/retina/templates/configmap.yaml b/deploy/standard/manifests/controller/helm/retina/templates/configmap.yaml
@@ -24,6 +24,7 @@ data:
     enableAnnotations: {{ .Values.enableAnnotations }}
     bypassLookupIPOfInterest: {{ .Values.bypassLookupIPOfInterest }}
     dataAggregationLevel: {{ .Values.dataAggregationLevel }}
+    telemetryInterval: {{ .Values.daemonset.telemetryInterval }}
 {{- end}}
 ---
 {{- if .Values.os.windows}}
@@ -48,6 +49,7 @@ data:
     enableTelemetry: {{ .Values.enableTelemetry }}
     enablePodLevel: {{ .Values.enablePodLevel }}
     remoteContext: {{ .Values.remoteContext }}
+    telemetryInterval: {{ .Values.daemonset.telemetryInterval }}
 {{- end}}
 
 

diff --git a/deploy/standard/manifests/controller/helm/retina/templates/operator.yaml b/deploy/standard/manifests/controller/helm/retina/templates/operator.yaml
@@ -307,6 +307,7 @@ data:
     captureDebug: {{ .Values.capture.debug }}
     captureJobNumLimit: {{ .Values.capture.jobNumLimit }}
     enableManagedStorageAccount: {{ .Values.capture.enableManagedStorageAccount }}
+    telemetryInterval: {{ .Values.operator.telemetryInterval }}
 {{- if .Values.capture.enableManagedStorageAccount }}
     azureCredentialConfig: /etc/cloud-config/azure.json
 {{- end }}

diff --git a/deploy/standard/manifests/controller/helm/retina/values.yaml b/deploy/standard/manifests/controller/helm/retina/values.yaml
@@ -27,6 +27,7 @@ operator:
     args:
       - "--config"
       - "/retina/operator-config.yaml"
+  telemetryInterval: "5m"
 
 image:
   repository: ghcr.io/microsoft/retina/retina-agent
@@ -87,6 +88,7 @@ daemonset:
       metricsBindAddress: ":18080"
       ports:
         containerPort: 10093
+  telemetryInterval: "15m"
 
 # volume mounts with name and mountPath
 volumeMounts:

diff --git a/go.mod b/go.mod
@@ -48,18 +48,18 @@ require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect
 	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
@@ -254,7 +254,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/spf13/pflag v1.0.5
+	github.com/spf13/pflag v1.0.6
 	github.com/stretchr/testify v1.10.0
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.34.0 // indirect
@@ -289,10 +289,10 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
 	github.com/Microsoft/hcsshim v0.12.0-rc.3
 	github.com/Sytten/logrus-zap-hook v0.1.0
-	github.com/aws/aws-sdk-go-v2 v1.34.0
-	github.com/aws/aws-sdk-go-v2/config v1.29.1
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.54
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1
+	github.com/aws/aws-sdk-go-v2 v1.36.0
+	github.com/aws/aws-sdk-go-v2/config v1.29.4
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.57
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2
 	github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5
 	github.com/cilium/cilium v1.16.0-pre.1.0.20240403152809-b9853ecbcaeb
 	github.com/cilium/ebpf v0.16.0