diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 157f09597b..7b350d71e8 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -13,7 +13,6 @@ permissions: jobs: e2e: - if: ${{ github.event_name == 'merge_group' }} name: E2E runs-on: ubuntu-latest @@ -29,7 +28,6 @@ jobs: - name: Az CLI login uses: azure/login@v2 - if: ${{ github.event_name == 'merge_group' }} with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -42,4 +40,4 @@ jobs: shell: bash run: | set -euo pipefail - go test -v ./test/e2e/. -timeout 60m -tags=e2e -count=1 -args -image-tag=$(make version) -image-registry=${{vars.ACR_NAME}} -image-namespace=${{github.repository}} + go test -v ./test/e2e/. -timeout 60m -tags=e2e -count=1 -args -image-tag=$(make version) -image-registry=${{vars.ACR_NAME}} -image-namespace=${{github.repository}} \ No newline at end of file diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 4089df0fef..187a48d658 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -23,9 +23,9 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v4.2.2 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v5.3.0 with: go-version-file: go.mod - run: go version @@ -35,7 +35,6 @@ jobs: - name: Az CLI login uses: azure/login@v2 - if: ${{ github.event_name == 'merge_group' }} with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -46,18 +45,13 @@ jobs: run: | set -euo pipefail echo "TAG=$(make version)" >> $GITHUB_ENV - if [ "$IS_MERGE_GROUP" == "true" ]; then - az acr login -n ${{ vars.ACR_NAME }} - make retina-image \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ - IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ - BUILDX_ACTION=--push - else - make retina-image \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} - fi + az acr login -n ${{ vars.ACR_NAME }} + make retina-image \ + IMAGE_NAMESPACE=${{ github.repository }} \ + PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ + IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ + APP_INSIGHTS_ID=${{ secrets.AZURE_APP_INSIGHTS_KEY }} \ + BUILDX_ACTION=--push env: IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }} @@ -73,9 +67,9 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v4.2.2 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v5.3.0 with: go-version-file: go.mod - run: go version @@ -85,7 +79,6 @@ jobs: - name: Az CLI login uses: azure/login@v2 - if: ${{ github.event_name == 'merge_group' }} with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -96,20 +89,14 @@ jobs: run: | set -euo pipefail echo "TAG=$(make version)" >> $GITHUB_ENV - if [ "$IS_MERGE_GROUP" == "true" ]; then - az acr login -n ${{ vars.ACR_NAME }} - make retina-image-win \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ - IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ - WINDOWS_YEARS=${{ matrix.year }} \ - BUILDX_ACTION=--push - else - make retina-image-win \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ - WINDOWS_YEARS=${{ matrix.year }} - fi + az acr login -n ${{ vars.ACR_NAME }} + make retina-image-win \ + IMAGE_NAMESPACE=${{ github.repository }} \ + PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ + IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ + APP_INSIGHTS_ID=${{ secrets.AZURE_APP_INSIGHTS_KEY }} \ + WINDOWS_YEARS=${{ matrix.year }} \ + BUILDX_ACTION=--push env: IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }} @@ -124,9 +111,9 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v4.2.2 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v5.3.0 with: go-version-file: go.mod - run: go version @@ -136,7 +123,6 @@ jobs: - name: Az CLI login uses: azure/login@v2 - if: ${{ github.event_name == 'merge_group' }} with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -147,18 +133,13 @@ jobs: run: | set -euo pipefail echo "TAG=$(make version)" >> $GITHUB_ENV - if [ "$IS_MERGE_GROUP" == "true" ]; then - az acr login -n ${{ vars.ACR_NAME }} - make retina-operator-image \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ - IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ - BUILDX_ACTION=--push - else - make retina-operator-image \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} - fi + az acr login -n ${{ vars.ACR_NAME }} + make retina-operator-image \ + IMAGE_NAMESPACE=${{ github.repository }} \ + PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ + IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ + APP_INSIGHTS_ID=${{ secrets.AZURE_APP_INSIGHTS_KEY }} \ + BUILDX_ACTION=--push env: IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }} @@ -173,9 +154,9 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v4.2.2 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v5.3.0 with: go-version-file: go.mod - run: go version @@ -185,7 +166,6 @@ jobs: - name: Az CLI login uses: azure/login@v2 - if: ${{ github.event_name == 'merge_group' }} with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -196,18 +176,12 @@ jobs: run: | set -euo pipefail echo "TAG=$(make version)" >> $GITHUB_ENV - if [ "$IS_MERGE_GROUP" == "true" ]; then - az acr login -n ${{ vars.ACR_NAME }} - make retina-shell-image \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ - IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ - BUILDX_ACTION=--push - else - make retina-shell-image \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} - fi + az acr login -n ${{ vars.ACR_NAME }} + make retina-shell-image \ + IMAGE_NAMESPACE=${{ github.repository }} \ + PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ + IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ + BUILDX_ACTION=--push env: IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }} @@ -222,9 +196,9 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v4.2.2 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v5.3.0 with: go-version-file: go.mod - run: go version @@ -234,7 +208,6 @@ jobs: - name: Az CLI login uses: azure/login@v2 - if: ${{ github.event_name == 'merge_group' }} with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -245,23 +218,17 @@ jobs: run: | set -euo pipefail echo "TAG=$(make version)" >> $GITHUB_ENV - if [ "$IS_MERGE_GROUP" == "true" ]; then - az acr login -n ${{ vars.ACR_NAME }} - make kubectl-retina-image \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ - IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ - BUILDX_ACTION=--push - else - make kubectl-retina-image \ - IMAGE_NAMESPACE=${{ github.repository }} \ - PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} - fi + az acr login -n ${{ vars.ACR_NAME }} + make kubectl-retina-image \ + IMAGE_NAMESPACE=${{ github.repository }} \ + PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \ + IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ + BUILDX_ACTION=--push env: IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }} manifests: - if: ${{ github.event_name == 'merge_group' && success('retina-images') && success('retina-win-images') && success('operator-images') && success('retina-shell-images')}} + if: ${{ success('retina-images') && success('retina-win-images') && success('operator-images') && success('retina-shell-images') && success('kubectl-retina-images') }} name: Generate Manifests runs-on: ubuntu-latest needs: @@ -279,7 +246,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v4.2.2 - name: Setup QEMU uses: docker/setup-qemu-action@v3 @@ -297,26 +264,26 @@ jobs: set -euo pipefail az acr login -n ${{ vars.ACR_NAME }} make manifest COMPONENT=${{ matrix.components }} \ - IMAGE_REGISTRY=${{ vars.ACR_NAME }} \ + IMAGE_REGISTRY=${{ vars.ACR_NAME }} e2e: + if: ${{ success('manifests') }} name: Run E2E Tests runs-on: ubuntu-latest needs: [manifests] steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v4.2.2 - name: Setup go - uses: actions/setup-go@v5 + uses: actions/setup-go@v5.3.0 with: go-version-file: go.mod - run: go version - name: Az CLI login uses: azure/login@v2 - if: ${{ github.event_name == 'merge_group' }} with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -329,38 +296,4 @@ jobs: shell: bash run: | set -euo pipefail - go test -v ./test/e2e/. -timeout 60m -tags=e2e -count=1 -args -image-tag=$(make version) -image-registry=${{ vars.ACR_NAME }} -image-namespace=${{ github.repository}} - - perf: - if: ${{ github.event_name == 'merge_group' && success('manifests')}} - name: Retina Performance Test - runs-on: ubuntu-latest - needs: [manifests] - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - run: go version - - - name: Az CLI login - uses: azure/login@v2 - if: ${{ github.event_name == 'merge_group' }} - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION }} - - - name: Run Perf Tests - env: - AZURE_APP_INSIGHTS_KEY: ${{ secrets.AZURE_APP_INSIGHTS_KEY }} - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION }} - AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} - shell: bash - run: | - set -euo pipefail - go test -v ./test/e2e/. -timeout 2h -tags=perf -count=1 -args -image-tag=$(make version) -image-registry=${{ vars.ACR_NAME }} -image-namespace=${{ github.repository }} + go test -v ./test/e2e/. -timeout 60m -tags=e2e -count=1 -args -image-tag=$(make version) -image-registry=${{ vars.ACR_NAME }} -image-namespace=${{ github.repository}} \ No newline at end of file diff --git a/controller/Dockerfile b/controller/Dockerfile index a8ae7f8e6e..f7b46b42cc 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -11,7 +11,7 @@ FROM --platform=$TARGETPLATFORM mcr.microsoft.com/cbl-mariner/base/core@sha256:7 # mcr.microsoft.com/cbl-mariner/distroless/minimal:2.0 FROM --platform=$TARGETPLATFORM mcr.microsoft.com/cbl-mariner/distroless/minimal@sha256:db87903c5d4d9d6760e86a274914efd6a3bb5914c0b5a6c6b35350ec297fea4f AS mariner-distroless -# mcr.microsoft.com/windows/servercore:ltsc2019 +# mcr.microsoft.com/windows/servercore:ltsc2019 FROM --platform=$TARGETPLATFORM mcr.microsoft.com/windows/servercore@sha256:6fdf140282a2f809dae9b13fe441635867f0a27c33a438771673b8da8f3348a4 AS ltsc2019 # mcr.microsoft.com/windows/servercore:ltsc2022 @@ -21,7 +21,7 @@ FROM --platform=$TARGETPLATFORM mcr.microsoft.com/windows/servercore@sha256:4595 # build stages # intermediate go generate stage -FROM golang AS intermediate +FROM golang AS intermediate ARG APP_INSIGHTS_ID # set to enable AI telemetry ARG GOARCH=amd64 # default to amd64 ARG GOOS=linux # default to linux @@ -32,6 +32,7 @@ RUN if [ "$GOOS" = "linux" ] ; then \ fi COPY ./pkg/plugin /go/src/github.com/microsoft/retina/pkg/plugin WORKDIR /go/src/github.com/microsoft/retina +RUN dir /go/src/github.com/microsoft/retina/pkg/plugin/ebpfwindows RUN if [ "$GOOS" = "linux" ] ; then \ go mod init github.com/microsoft/retina; \ go generate -skip "mockgen" -x /go/src/github.com/microsoft/retina/pkg/plugin/...; \ @@ -64,7 +65,7 @@ ARG GOOS=linux # default to linux ARG VERSION ENV GOARCH=${GOARCH} ENV GOOS=${GOOS} -RUN --mount=type=cache,target="/root/.cache/go-build" go build -v -o /go/bin/retina/controller -ldflags "-X github.com/microsoft/retina/internal/buildinfo.Version="$VERSION" -X github.com/microsoft/retina/internal/buildinfo.ApplicationInsightsID="$APP_INSIGHTS_ID"" controller/main.go +RUN --mount=type=cache,target="/root/.cache/go-build" go build -v -o /go/bin/retina/controller -ldflags "-X github.com/microsoft/retina/internal/buildinfo.Version="$VERSION" -X github.com/microsoft/retina/internal/buildinfo.ApplicationInsightsID="$APP_INSIGHTS_ID"" controller/main.go # init binary @@ -115,7 +116,6 @@ COPY --from=tools /lib/ /lib COPY --from=tools /usr/lib/ /usr/lib ENTRYPOINT ["./retina/initretina"] - # agent final image # mcr.microsoft.com/cbl-mariner/distroless/minimal:2.0 # mcr.microsoft.com/cbl-mariner/distroless/minimal@sha256:63a0a70ceaa1320bc6eb98b81106667d43e46b674731ea8d28e4de1b87e0747f @@ -133,7 +133,7 @@ ENV HUBBLE_SERVER=unix:///var/run/cilium/hubble.sock ENTRYPOINT ["./retina/controller"] -# agent final image for windows +# agent final image for windows FROM ${OS_VERSION} AS agent-win COPY --from=controller-bin /go/src/github.com/microsoft/retina/windows/kubeconfigtemplate.yaml kubeconfigtemplate.yaml COPY --from=controller-bin /go/src/github.com/microsoft/retina/windows/setkubeconfigpath.ps1 setkubeconfigpath.ps1 diff --git a/deploy/legacy/manifests/controller/helm/retina/values.yaml b/deploy/legacy/manifests/controller/helm/retina/values.yaml index fca1c1822d..7a0e6e87ba 100644 --- a/deploy/legacy/manifests/controller/helm/retina/values.yaml +++ b/deploy/legacy/manifests/controller/helm/retina/values.yaml @@ -10,8 +10,8 @@ os: operator: name: retina-operator enabled: false - repository: ghcr.io/microsoft/retina/retina-operator - tag: "v0.0.2" + repository: demoebpfregistry.azurecr.io/vpidatala94/retina/retina-operator + tag: "a6d7eb3" installCRDs: true enableRetinaEndpoint: false resources: @@ -29,11 +29,11 @@ operator: - "/retina/operator-config.yaml" image: - repository: ghcr.io/microsoft/retina/retina-agent - initRepository: ghcr.io/microsoft/retina/retina-init + repository: demoebpfregistry.azurecr.io/vpidatala94/retina/retina-agent + initRepository: demoebpfregistry.azurecr.io/vpidatala94/retina/retina-init pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "v0.0.2" + tag: "a6d7eb3" enableConntrackMetrics: false enablePodLevel: false @@ -64,7 +64,7 @@ apiServer: logLevel: debug enabledPlugin_linux: '["dropreason","packetforward","linuxutil","dns"]' -enabledPlugin_win: '["hnsstats", "enabledPlugin_win"]' +enabledPlugin_win: '["hnsstats", "ebpfwindows"]' enableTelemetry: false diff --git a/go.mod b/go.mod index a47b318b2e..b9c8d1bed5 100644 --- a/go.mod +++ b/go.mod @@ -346,5 +346,4 @@ require ( ) replace github.com/vishvananda/netns => github.com/inspektor-gadget/netns v0.0.5-0.20230524185006-155d84c555d6 - replace k8s.io/perf-tests/network/benchmarks/netperf => github.com/Azure/perf-tests/network/benchmarks/netperf v0.0.0-20241008140716-395a79947d2c diff --git a/pkg/plugin/ebpfwindows/ebpf_windows.go b/pkg/plugin/ebpfwindows/ebpf_windows.go index b3798e7ddb..a989c36139 100644 --- a/pkg/plugin/ebpfwindows/ebpf_windows.go +++ b/pkg/plugin/ebpfwindows/ebpf_windows.go @@ -26,7 +26,7 @@ import ( const ( // name of the ebpfwindows plugin - name string = "windowseBPF" + name string = "ebpfwindows" // name of the metrics packetsReceived string = "win_packets_recv_count" packetsSent string = "win_packets_sent_count" diff --git a/pkg/plugin/ebpfwindows/event_writer/event_writer_util.cpp b/pkg/plugin/ebpfwindows/event_writer/event_writer_util.cpp deleted file mode 100644 index 72a656efcb..0000000000 --- a/pkg/plugin/ebpfwindows/event_writer/event_writer_util.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include -#include -#include -#include - -int pin_map(const char* pin_path, bpf_map* map) { - int map_fd = 0; - // Attempt to open the pinned map - map_fd = bpf_obj_get(pin_path); - if (map_fd < 0) { - // Get the file descriptor of the map - map_fd = bpf_map__fd(map); - - if (map_fd < 0) { - fprintf(stderr, "%s - failed to get map file descriptor\n", __FUNCTION__); - return 1; - } - - if (bpf_obj_pin(map_fd, pin_path) < 0) { - fprintf(stderr, "%s - failed to pin map to %s\n", pin_path, __FUNCTION__); - return 1; - } - - printf("%s - map successfully pinned at %s\n", pin_path, __FUNCTION__); - } else { - printf("%s -pinned map found at %s\n", pin_path, __FUNCTION__); - } - return 0; -} \ No newline at end of file diff --git a/pkg/plugin/ebpfwindows/event_writer/event_writer_util.h b/pkg/plugin/ebpfwindows/event_writer/event_writer_util.h deleted file mode 100644 index 973af0c5e3..0000000000 --- a/pkg/plugin/ebpfwindows/event_writer/event_writer_util.h +++ /dev/null @@ -1,6 +0,0 @@ -#include -#include -#include -#include - -int pin_map(const char* pin_path, bpf_map* map); \ No newline at end of file diff --git a/pkg/plugin/ebpfwindows/retinaebpfapi.lib b/pkg/plugin/ebpfwindows/retinaebpfapi.lib new file mode 100644 index 0000000000..4f94b94218 Binary files /dev/null and b/pkg/plugin/ebpfwindows/retinaebpfapi.lib differ diff --git a/pkg/plugin/include_windows.go b/pkg/plugin/include_windows.go index 1cbb240eca..294d6066da 100644 --- a/pkg/plugin/include_windows.go +++ b/pkg/plugin/include_windows.go @@ -5,4 +5,5 @@ package plugin import ( _ "github.com/microsoft/retina/pkg/plugin/hnsstats" _ "github.com/microsoft/retina/pkg/plugin/pktmon" + _ "github.com/vpidatala94/retina/pkg/plugin/ebpfwindows" ) diff --git a/scripts/install-ebpf-xdp.ps1 b/scripts/install-ebpf-xdp.ps1 new file mode 100644 index 0000000000..d5a68d5a6d --- /dev/null +++ b/scripts/install-ebpf-xdp.ps1 @@ -0,0 +1,805 @@ +#Requires -RunAsAdministrator + +Function Assert-SoftwareInstalled +{ + [cmdletbinding(DefaultParameterSetName='Software')] + + Param + ( + [Parameter(ParameterSetName='Service',Mandatory=$true)] + [ValidateScript({-Not [String]::IsNullOrWhiteSpace($_)})] + [String] $ServiceName, + + [Parameter(ParameterSetName='Service',Mandatory=$false)] + [ValidateSet($null,'Running','Stopped')] + [String] $ServiceState, + + [Parameter(ParameterSetName='Software',Mandatory=$true)] + [ValidateScript({-Not [String]::IsNullOrWhiteSpace($_)})] + [String] $SoftwareName, + + [Parameter(ParameterSetName='Software',Mandatory=$false)] + [String] $SoftwareVersion, + + [Parameter(ParameterSetName='Service',Mandatory=$false)] + [Parameter(ParameterSetName='Software',Mandatory=$false)] + [Switch] $Silent + ) + + [String] $name = If($ServiceName) {"$($ServiceName)"}Else{"$($SoftwareName)"} + + If(-Not $Silent.IsPresent) + { + Write-Host -Object:"Checking if $($name) is installed ..." + } + + [Boolean] $isInstalled = $false + + Try + { + If($SoftwareName) + { + $software = Get-WmiObject -Class:'Win32_Product' | Where-Object -Property:'Name' -like "*$($SoftwareName)*" + + If($software -And + (-Not [String]::IsNullOrWhiteSpace($SoftwareVersion))) + { + $software = $software | Where-Object -Property:'Version' -like "*$($SoftwareVersion)*" + } + + If($software) + { + $isInstalled = $true + } + } + ElseIF($ServiceName) + { + [Object] $state = Get-Service -Name:"$($ServiceName)" -ErrorAction:'SilentlyContinue' + If($state) + { + $isInstalled = $true + + If($ServiceState -And + -Not ($state.Status -INE $ServiceState)) + { + Write-Warning -Message:"`t$ServiceName is $$(state.Status)" + } + } + } + } + Catch + { + + } + + If(-Not $Silent.IsPresent) + { + If($isInstalled) + { + Write-Host -Object:"`t$($name) is installed" + } + Else + { + Write-Host -Object:"`t$($name) is not installed" + } + } + + Return $isInstalled +} + +<# + .Name + Assert-TestSigningIsEnabled + + .Synopsis + Internal cmdlet to check if testsigning is enabled in the boot loader. + + .Description + Returns TRUE if test signing is enabled, otherwise FALSE. + + .Parameter Silent + Optional switch used to suppress output messages + + .Example + # Check if test signing is enabled + Assert-TestSigningIsEnabled +#> +Function Assert-TestSigningIsEnabled +{ + Param + ( + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [Switch] $Silent + ) + + [Boolean] $isEnabled = $false + [String] $state = 'Disabled' + + Try + { + [Boolean] $current = $false + + If(-Not ($Silent.IsPresent)) + { + Write-Host -Object:"`tAssert Test Signing is Enabled" + } + + [Object[]] $entries = BCDEdit.exe /enum + If($entries.Count -ILT 3) + { + Write-Error -Message:"$entries" + + Throw + } + + ForEach($entry in $entries) + { + If($entry.StartsWith('identifier')) + { + If($entry -ILike '*{current}*') + { + $current = $true + } + Else + { + $current = $false + } + } + Else + { + If($current) + { + If($entry -ILike '*testsigning*Yes*') + { + $state = 'Enabled' + + $isEnabled = $true + + Break + } + } + } + } + } + Catch + { + $isEnabled = $false + + $state = 'Unknown' + } + + If(-Not ($Silent.IsPresent)) + { + Write-Host -Object:"`t`t$($state)" + } + + Return $isEnabled +} + +<# + .Name + Disable-TestSigning + + .Synopsis + Internal cmdlet to turn off Test Signing in the Windows Boot Loader. + + .Description + Returns TRUE if test signing is disabled, otherwise FALSE. + If set, the setting does not take effect until a reboot + + .Parameter Reboot + Optional parameter which will trigger a reboot if needed + + .Example + # Disable test signing + Disable-TestSigning +#> +Function Disable-TestSigning +{ + Param + ( + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [Switch] $Reboot + ) + + [Boolean] $isSuccess = $true + + Try + { + [Boolean] $current = $false + [Boolean] $found = $false + + Write-Host -Object:"`tDisabling Test Signing" + + If(Assert-TestSigningIsEnabled -Silent) + { + Start-Process -FilePath:"$($env:WinDir)\System32\BCDEdit.exe" -ArgumentList @('/Set TestSigning Off') -PassThru | Wait-Process + + If(Assert-TestSigningIsEnabled -Silent) + { + Write-Error -Message:"`t`tFailed" + + Throw + } + + $script:RequiresReboot = $true + } + + Write-Host -Object:"`t`tDisabled" + } + Catch + { + $isSuccess = $false + } + + If($Reboot.IsPresent -and + $script:RequiresReboot) + { + Write-Host -Object:'Restarting' + + Start-Sleep -Seconds:5 + + Restart-Computer + } + + Return $isSuccess +} + +<# + .Name + Enable-TestSigning + + .Synopsis + Internal cmdlet to turn on Test Signing in the Windows Boot Loader. + + .Description + Returns TRUE if test signing is enabled, otherwise FALSE. + + .Parameter Reboot + Optional parameter which will trigger a reboot if needed + + .Example + # Enable test signing + Enable-TestSigning +#> +Function Enable-TestSigning +{ + Param + ( + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [Switch] $Reboot + ) + + [Boolean] $isSuccess = $true + + Try + { + [Boolean] $current = $false + [Boolean] $found = $false + + Write-Host -Object:"`tEnabling Test Signing" + + If(-Not (Assert-TestSigningIsEnabled -Silent)) + { + Start-Process -FilePath:"$($env:WinDir)\System32\BCDEdit.exe" -ArgumentList @('/Set TestSigning On') -PassThru | Wait-Process + + If(-Not (Assert-TestSigningIsEnabled -Silent)) + { + Write-Error -Message:"`t`tFailed" + + Throw + } + + $script:RequiresReboot = $true + } + + Write-Host -Object:"`t`tEnabled" + + } + Catch + { + $isSuccess = $false + } + + If($Reboot.IsPresent -and + $script:RequiresReboot) + { + Write-Host -Object:'Restarting' + + Start-Sleep -Seconds:5 + + Restart-Computer + } + + Return $isSuccess +} + +#endregion PrivateFns + +#region Public + +<# + .Name + Assert-WindowsCiliumIsReady + + .Synopsis + Check if Cilium for Windows is ready + + .Description + Returns TRUE if Cilium for Windows is ready, otherwise FALSE. + + .Example + # Check if Cilium for Windows is ready + Assert-WindowsCiliumFunctions +#> +Function Assert-WindowsCiliumIsReady +{ + Write-Host -Object:'Validating Cilium for Windows is ready' + + [Boolean] $isReady = $true + [String[]] $services = @( + 'eBPFCore', + 'NetEbpfExt', + 'XDP' + ) + ForEach($service in $services) + { + If(-Not (Assert-SoftwareInstalled -ServiceName:"$($service)" -ServiceState:'Running')) + { + $isReady = $false + + Write-Warning -Message:"`t$($service) is not ready" + } + } + + Return $isReady +} + +<# + .Name + Install-eBPF + + .Synopsis + Installs extended Berkley Packet Filter for Windows. + + .Description + Returns TRUE if extended Berkley Packet Filter for Windows is installed successfully, otherwise FALSE. + Function requires that Test Signing is enabled. + + .Parameter LocalPath + Local directory to the eBPF for Windows binaries. + Default location is $env:LocalAppData\Temp + + .Example + # Install eBPF for Windows + Install-eBPF -LocalPath:"$env:TEMP" +#> +Function Install-eBPF +{ + [cmdletbinding(DefaultParameterSetName='Default')] + + Param + ( + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [ValidateScript({Test-Path $_ -PathType:'Container'})] + [String] $LocalPath = "$env:TEMP" + ) + + [Boolean] $isSuccess = $true + + Try + { + Write-Host 'Installing extended Berkley Packet Filter for Windows' + If(-Not (Assert-TestSigningIsEnabled)) + { + If(-Not (Enable-TestSigning -Reboot)) { Throw } + } + + If(Assert-SoftwareInstalled -ServiceName:"eBPFCore") + { + Write-Host 'extended Berkley Packet Filter for Windows is already installed' + return $isSuccess + } + + Write-Host 'Installing extended Berkley Packet Filter for Windows' + $ProgressPreference = 'SilentlyContinue' + # Download eBPF-for-Windows. + $packageEbpfUrl = "https://github.com/microsoft/ebpf-for-windows/releases/download/Release-v0.20.0/Build-x64-native-only.NativeOnlyRelease.zip" + Invoke-WebRequest -Uri $packageEbpfUrl -OutFile "$LocalPath\\Build-x64-native-only-NativeOnlyRelease.zip" + Expand-Archive -Path "$LocalPath\\Build-x64-native-only-NativeOnlyRelease.zip" -DestinationPath "$LocalPath\\Build-x64-native-only-NativeOnlyRelease\\msi" -Force + Start-Process -FilePath:"$($env:WinDir)\System32\MSIExec.exe" -ArgumentList @("/i $($LocalPath)\ebpf-for-windows.msi", '/qn', "INSTALLFOLDER=`"$($env:ProgramFiles)\ebpf-for-windows`"", 'ADDLOCAL=eBPF_Runtime_Components') -PassThru | Wait-Process + + If(-Not (Assert-SoftwareInstalled -ServiceName:'eBPFCore' -Silent) -Or + -Not (Assert-SoftwareInstalled -ServiceName:'NetEbpfExt' -Silent)) + { + Write-Error -Message:"`teBPF service failed to install" + + Throw + } + + $isSuccess = Assert-SoftwareInstalled -ServiceName:"eBPFCore" + } + Catch + { + $isSuccess = $false + Write-Host "EBPF install failed : $_" + Uninstall-eBPF + } + + Return $isSuccess +} + +<# + .Name + Install-XDP + + .Synopsis + Installs the eXpress Data Path for Windows service. + + .Description + Returns TRUE if the eXpress Data Path for Windows service is installed successfully, otherwise FALSE. + + .Parameter LocalPath + Local directory to the eXpress Data Path for Windows service binaries. + Default location is $env:LocalAppData\Temp + + .Example + # Install the eXpress Data Path service + Install-XDP -LocalPath:"$env:TEMP" +#> +Function Install-XDP +{ + [cmdletbinding(DefaultParameterSetName='Default')] + + Param + ( + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [ValidateScript({Test-Path $_ -PathType:'Container'})] + [String] $LocalPath = "$env:TEMP" + ) + + [Boolean] $isSuccess = $true + + Try + { + If(Assert-SoftwareInstalled -SoftwareName:'XDP for Windows' -Silent) + { + Write-Host 'XDP for Windows is already installed' + return $isSuccess + } + + # Download XDP-for-Windows. + Write-Host 'Installing eXpress Data Path for Windows' + $packageXdpUrl = "https://github.com/microsoft/xdp-for-windows/releases/download/v1.1.0%2Bbed474a/bin_Release_x64.zip" + Invoke-WebRequest -Uri $packageXdpUrl -OutFile "$LocalPath\\bin_Release_x64.zip" + Expand-Archive -Path "$LocalPath\\bin_Release_x64.zip" -DestinationPath "$LocalPath\\bin_Release_x64" -Force + copy "$LocalPath\\bin_Release_x64\\amd64fre\\xdp.cer" $LocalPath + copy "$LocalPath\\bin_Release_x64\\amd64fre\\xdpcfg.exe" $LocalPath + CertUtil.exe -addstore Root "$LocalPath\xdp.cer" + CertUtil.exe -addstore TrustedPublisher "$LocalPath\xdp.cer" + Invoke-WebRequest -Uri "https://github.com/microsoft/xdp-for-windows/releases/download/v1.1.0%2Bbed474a/xdp-for-windows.1.1.0.msi" -OutFile "$LocalPath\xdp-for-windows.1.1.0.msi" + Start-Process -FilePath:"$($env:WinDir)\System32\MSIExec.exe" -ArgumentList @("/i $LocalPath\xdp-for-windows.1.1.0.msi", '/qn') -PassThru | Wait-Process + sc.exe query xdp + reg.exe add "HKLM\SYSTEM\CurrentControlSet\Services\xdp\Parameters" /v XdpEbpfEnabled /d 1 /t REG_DWORD /f + net.exe stop xdp + net.exe start xdp + Write-Output "XDP for Windows installed" + Write-Host "Setting SDDL for XDP service" + & "$LocalPath\xdpcfg.exe" SetDeviceSddl "D:P(A;;GA;;;SY)(A;;GA;;;BA)" + If(-Not (Assert-SoftwareInstalled -SoftwareName:'XDP for Windows' -Silent)) { + Throw + } + } + Catch + { + $isSuccess = $false + Write-Host "XDP install failed : $_" + Uninstall-XDP + } + + Return $isSuccess +} + +<# + .Name + Install-WindowsCilium + + .Description + Returns TRUE if the Cilium for Windows installation is successfully, otherwise FALSE. + Requires test signing to be enabled. This is asserted up front and may trigger a restart. + A restart will occur if the NoRestart switch is not specified. Any caller needs to account for this. +#> +Function Install-WindowsCilium +{ + [cmdletbinding(DefaultParameterSetName='Default')] + + [Boolean] $isSuccess = $true + + Try + { + If(Assert-WindowsCiliumIsReady) { + Write-Host 'Windows Cilium Installed already' + return + } + + Write-Host 'Installing Windows Cilium' + + If(-Not (Assert-TestSigningIsEnabled -Silent)) + { + If(-Not (Enable-TestSigning -Reboot)) { Throw } + } + + If(-Not (Install-eBPF)) {Throw} + + If(-Not (Install-XDP)) {Throw} + + # Create the probe ready file + New-Item -Path "C:\install-ebpf-xdp-probe-ready" -ItemType File -Force + + Write-Host -Object:'Restarting' + + Start-Sleep -Seconds:5 + + If(Assert-WindowsCiliumIsReady) { + Throw "Cilium for Windows is not ready" + } + + Restart-Computer + } + Catch + { + Write-Host "Exception: $_" + $isSuccess = $false + } + + Return $isSuccess +} + +<# + .Name + Uninstall-eBPF + + .Synopsis + Uninstalls the extended Berkley Packet Filter for Windows. + + .Description + Returns TRUE if the extended Berkley Packet Filter for Windows is uninstalled successfully, otherwise FALSE. + + .Parameter LocalPath + Local directory to the extended Berkley Packet Filter for Windows binaries. + Default location is $env:LocalAppData\Temp + + .Example + # Uninstall the extended Berkley Packet Filter for Windows + Uninstall-eBPF -LocalPath:"$(env:LocalAppData)\Temp" +#> +Function Uninstall-eBPF +{ + [cmdletbinding(DefaultParameterSetName='Default')] + + Param + ( + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [ValidateScript({Test-Path $_ -PathType:'Container'})] + [String] $LocalPath = "$env:TEMP" + ) + + Write-Host 'Uninstalling the extended Berkley Packet Filter for Windows' + + [Boolean] $isSuccess = $true + + Try + { + [String[]] $services = @('eBPFCore', + 'NetEbpfExt' + ) + + ForEach($service in $services) + { + [Object] $state = Get-Service -Name:$($service) -ErrorAction:'SilentlyContinue' + If($state) + { + For([Byte]$i = 0; + $i -ILE 5; + $i++) + { + If($state.Status -IEQ 'Stopped') + { + Break + } + Else + { + If($state.Status -IEQ 'Running') + { + Stop-Service -Name:"$($service)" -Force + } + ElseIf($state.Status -IEQ 'StopPending') + { + Start-Sleep -Seconds:5 + } + Else + { + Write-Error -Message:"$($service) service is $($state.status)" + + Throw + } + } + + $state = Get-Service -Name:"$($service)" + } + } + + Start-Process -FilePath:"$($env:WinDir)\System32\MSIExec.exe" -ArgumentList @("/x $($LocalPath)\ebpf-for-windows.msi", '/qn') -PassThru | Wait-Process + } + + If((Assert-SoftwareInstalled -ServiceName:'eBPFCore' -Silent) -or + (Assert-SoftwareInstalled -ServiceName:'NetEbpfExt' -Silent) -or + (Assert-SoftwareInstalled -SoftwareName:'eBPF for Windows' -Silent)) + { + Write-Error -Message:"eBPF for Windows is still installed" + + Throw + } + } + Catch + { + $isSuccess = $false + } + + Return $isSuccess +} + +<# + .Name + Uninstall-XDP + + .Synopsis + Uninstalls the express Data Path for Windows service + + .Description + Returns TRUE if the eXpress Data Path for Windows service is uninstalled successfully, otherwise FALSE. + + .Parameter LocalPath + Local directory to the eXpress Data Path for Windows service binaries. + Default location is $env:LocalAppData\Temp + + .Example + # Uninstall the eXpress Data Path for Windows service + Uninstall-XDP -LocalPath:"$($env:LocalAppData)\Temp" +#> +Function Uninstall-XDP +{ + [cmdletbinding(DefaultParameterSetName='Default')] + + Param + ( + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [ValidateScript({Test-Path $_ -PathType:'Container'})] + [String] $LocalPath = "$env:TEMP" + ) + + Write-Host 'Uninstalling eXpress Data Path for Windows' + + [Boolean] $isSuccess = $true + + Try + { + [Object] $state = Get-Service -Name:'XDP' -ErrorAction:'SilentlyContinue' + If($state) + { + For([Byte]$i = 0; + $i -ILE 5; + $i++) + { + If($state.Status -IEQ 'Stopped') + { + Break + } + Else + { + If($state.Status -IEQ 'Running') + { + Stop-Service -Name:'XDP' -Force + } + ElseIf($state.Status -IEQ 'StopPending') + { + Start-Sleep -Seconds:5 + } + Else + { + Write-Error -Message:"XDP service is $($state.status)" + + Throw + } + } + + $state = Get-Service -Name:'XDP' + } + + $regValue = New-ItemProperty -Path:'HKLM:\SYSTEM\CurrentControlSet\Services\xdp\Parameters' -Name:'xdpEbpfEnabled' -PropertyType:'DWORD' -Value:0 -Force + If($regValue.xdpEbpfEnabled -IEQ 0) + { + Start-Process -FilePath:"$($env:WinDir)\System32\MSIExec.exe" -ArgumentList @("/x $($LocalPath)\xdp-for-windows.1.1.0.msi", '/qn') -PassThru | Wait-Process + } + } + + If((Assert-SoftwareInstalled -ServiceName:'XDP' -Silent) -or + (Assert-SoftwareInstalled -SoftwareName:'XDP for Windows' -Silent)) + { + Write-Error -Message:"XDP for Windows is still installed" + + Throw + } + } + Catch + { + $isSuccess = $false + } + + Return $isSuccess +} + +<# + .Name + Uninstall-WindowsCilium + + .Synopsis + Uninstalls Cilium for Windows + + .Parameter LocalPath + Local directory to the Cilium for Windows binaries. + Default location is $env:LocalAppData\Temp + + .Parameter DisableTestSigning + Optional switch used to disable test signing on the current Windows boot loader + + .Description + Returns TRUE if Cilium for Windows is uninstalled successfully, otherwise FALSE. + + .Example + # Uninstall Cilium for Windows + Uninstall-WindowsCilium +#> +Function Uninstall-WindowsCilium +{ + [cmdletbinding(DefaultParameterSetName='Default')] + + Param + ( + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [ValidateScript({Test-Path $_ -PathType:'Container'})] + [String] $LocalPath = "$env:TEMP", + + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [Switch] $DisableTestSigning, + + [Parameter(ParameterSetName='Default',Mandatory=$false)] + [Switch] $NoReboot + ) + + Write-Host 'Uninstalling Cilium for Windows' + + [Boolean] $isSuccess = $true + + Try + { + Uninstall-XDP -LocalPath:"$($LocalPath)" + + Uninstall-eBPF -LocalPath:"$($LocalPath)" + + If($DisableTestSigning.IsPresent) + { + Disable-TestSigning -Reboot:$(-Not ($NoReboot.IsPresent)) + } + } + Catch + { + $isSuccess = $false + } + + Return $isSuccess +} + + +#Script Start +Install-WindowsCilium \ No newline at end of file diff --git a/test/e2e/framework/kubernetes/apply-yaml-config.go b/test/e2e/framework/kubernetes/apply-yaml-config.go new file mode 100644 index 0000000000..ee98cd2b99 --- /dev/null +++ b/test/e2e/framework/kubernetes/apply-yaml-config.go @@ -0,0 +1,107 @@ +package kubernetes + +import ( + "bytes" + "context" + "fmt" + "log" + "os" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + applyTimeout = 10 * time.Minute +) + +type ApplyYamlConfig struct { + KubeConfigFilePath string + YamlFilePath string +} + +func (a *ApplyYamlConfig) Run() error { + ctx, cancel := context.WithTimeout(context.Background(), applyTimeout) + defer cancel() + + config, err := clientcmd.BuildConfigFromFlags("", a.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating dynamic client: %w", err) + } + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + return fmt.Errorf("error creating discovery client: %w", err) + } + + resources, err := restmapper.GetAPIGroupResources(discoveryClient) + if err != nil { + return fmt.Errorf("error getting API group resources: %w", err) + } + + mapper := restmapper.NewDiscoveryRESTMapper(resources) + + yamlFile, err := os.ReadFile(a.YamlFilePath) + if err != nil { + return fmt.Errorf("error reading YAML file: %w", err) + } + + reader := bytes.NewReader(yamlFile) + decoder := yaml.NewYAMLOrJSONDecoder(reader, 100) + var rawObj unstructured.Unstructured + if err := decoder.Decode(&rawObj); err != nil { + return fmt.Errorf("error decoding YAML file: %w", err) + } + + // Get GroupVersionResource to invoke the dynamic client + gvk := rawObj.GroupVersionKind() + restMapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return fmt.Errorf("error getting REST mapping: %w", err) + } + gvr := restMapping.Resource + + // Apply the YAML document + namespace := rawObj.GetNamespace() + if len(namespace) == 0 { + namespace = "default" + } + applyOpts := metav1.ApplyOptions{FieldManager: "kube-apply"} + _, err = dynamicClient.Resource(gvr).Namespace(namespace).Apply(ctx, rawObj.GetName(), &rawObj, applyOpts) + if err != nil { + return fmt.Errorf("apply error: %w", err) + } + + log.Printf("applied YAML file: %s\n", a.YamlFilePath) + return nil +} + +func (a *ApplyYamlConfig) Prevalidate() error { + _, err := os.Stat(a.YamlFilePath) + if os.IsNotExist(err) { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current working directory %s: %w", cwd, err) + } + log.Printf("the current working directory %s", cwd) + return fmt.Errorf("YAML file not found at %s: working directory: %s: %w", a.YamlFilePath, cwd, err) + } + log.Printf("found YAML file at %s", a.YamlFilePath) + + return nil +} + +func (a *ApplyYamlConfig) Stop() error { + return nil +} diff --git a/test/e2e/jobs/jobs.go b/test/e2e/jobs/jobs.go index 375f69aa4e..89c393fa30 100644 --- a/test/e2e/jobs/jobs.go +++ b/test/e2e/jobs/jobs.go @@ -106,6 +106,32 @@ func UninstallRetina(kubeConfigFilePath, chartPath string) *types.Job { return job } +func InstallEbpfXdp(kubeConfigFilePath string) *types.Job { + job := types.NewJob("Install ebpf and xdp") + job.AddStep(&kubernetes.CreateNamespace{ + KubeConfigFilePath: kubeConfigFilePath, + Namespace: "ebpf-xdp-install"}, nil) + + job.AddStep(&kubernetes.ApplyYamlConfig{ + YamlFilePath: "yaml/windows/install-ebpf-xdp.yaml", + }, nil) + + return job +} + +func InstallEventWriter(kubeConfigFilePath string) *types.Job { + job := types.NewJob("Install event writer") + job.AddStep(&kubernetes.CreateNamespace{ + KubeConfigFilePath: kubeConfigFilePath, + Namespace: "install-event-writer"}, nil) + + job.AddStep(&kubernetes.ApplyYamlConfig{ + YamlFilePath: "yaml/windows/install-event-writer.yaml", + }, nil) + + return job +} + func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string, testPodNamespace string) *types.Job { job := types.NewJob("Install and test Retina with basic metrics") @@ -168,6 +194,7 @@ func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string, test } job.AddScenario(windows.ValidateWindowsBasicMetric()) + job.AddScenario(windows.ValidateCiliumBasicMetric()) } job.AddStep(&kubernetes.EnsureStableComponent{ diff --git a/test/e2e/retina_e2e_test.go b/test/e2e/retina_e2e_test.go index 546f596bb5..00eef7cef2 100644 --- a/test/e2e/retina_e2e_test.go +++ b/test/e2e/retina_e2e_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/microsoft/retina/test/e2e/common" "github.com/microsoft/retina/test/e2e/framework/helpers" @@ -57,21 +58,37 @@ func TestE2ERetina(t *testing.T) { createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath, *common.CreateInfra)) createTestInfra.Run(ctx) + // Install Ebpf and XDP + installEbpfAndXDP := types.NewRunner(t, jobs.InstallEbpfXdp(kubeConfigFilePath)) + installEbpfAndXDP.Run(ctx) + t.Cleanup(func() { if *common.DeleteInfra { _ = jobs.DeleteTestInfra(subID, rg, clusterName, location).Run() } }) + time.Sleep(10 * time.Minute) + + // Install Ebpf and XDP + installEventWriter := types.NewRunner(t, jobs.InstallEventWriter(kubeConfigFilePath)) + installEventWriter.Run(ctx) + + time.Sleep(10 * time.Minute) + // Install and test Retina basic metrics basicMetricsE2E := types.NewRunner(t, jobs.InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath, common.TestPodNamespace)) basicMetricsE2E.Run(ctx) - // Upgrade and test Retina with advanced metrics + time.Sleep(10 * time.Minute) + //Upgrade and test Retina with advanced metrics advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath, common.TestPodNamespace)) advanceMetricsE2E.Run(ctx) // Install and test Hubble basic metrics validatehubble := types.NewRunner(t, jobs.ValidateHubble(kubeConfigFilePath, hubblechartPath, common.TestPodNamespace)) validatehubble.Run(ctx) + + // Install and test Cilium Windows basics and advanced metrics + time.Sleep(10 * time.Minute) } diff --git a/test/e2e/scenarios/windows/cilium-metrics.go b/test/e2e/scenarios/windows/cilium-metrics.go new file mode 100644 index 0000000000..845a9defcc --- /dev/null +++ b/test/e2e/scenarios/windows/cilium-metrics.go @@ -0,0 +1,87 @@ +package windows + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubernetes "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +type ValidateCiliumMetric struct { + KubeConfigFilePath string + RetinaDaemonSetNamespace string + RetinaDaemonSetName string +} + +func (v *ValidateCiliumMetric) Run() error { + config, err := clientcmd.BuildConfigFromFlags("", v.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + pods, err := clientset.CoreV1().Pods(v.RetinaDaemonSetNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "k8s-app=retina", + }) + if err != nil { + panic(err.Error()) + } + + var windowsRetinaPod *v1.Pod + for pod := range pods.Items { + if pods.Items[pod].Spec.NodeSelector["kubernetes.io/os"] == "windows" { + windowsRetinaPod = &pods.Items[pod] + } + } + if windowsRetinaPod == nil { + return ErrorNoWindowsPod + } + /* + labels := map[string]string{ + "direction": "win_packets_sent_count", + } + + log.Printf("checking for metric %s with labels %+v\n", hnsMetricName, labels) + + // wrap this in a retrier because windows is slow + var output []byte + err = defaultRetrier.Do(context.TODO(), func() error { + output, err = k8s.ExecPod(context.TODO(), clientset, config, windowsRetinaPod.Namespace, windowsRetinaPod.Name, fmt.Sprintf("curl -s http://localhost:%d/metrics", common.RetinaPort)) + if err != nil { + return fmt.Errorf("error executing command in windows retina pod: %w", err) + } + if len(output) == 0 { + return ErrNoMetricFound + } + + if err != nil { + return fmt.Errorf("failed to get metrics from windows retina pod: %w", err) + } + + err = prom.CheckMetricFromBuffer(output, hnsMetricName, labels) + if err != nil { + return fmt.Errorf("failed to verify prometheus metrics: %w", err) + } + + return nil + }) + + log.Printf("found metric matching %+v: with labels %+v\n", hnsMetricName, labels) + */ + return nil +} + +func (v *ValidateCiliumMetric) Prevalidate() error { + return nil +} + +func (v *ValidateCiliumMetric) Stop() error { + return nil +} diff --git a/test/e2e/scenarios/windows/scenario.go b/test/e2e/scenarios/windows/scenario.go index a452a04010..b1443c6679 100644 --- a/test/e2e/scenarios/windows/scenario.go +++ b/test/e2e/scenarios/windows/scenario.go @@ -18,3 +18,17 @@ func ValidateWindowsBasicMetric() *types.Scenario { } return types.NewScenario(name, steps...) } + +func ValidateCiliumBasicMetric() *types.Scenario { + name := "Cilium Windows Metrics" + steps := []*types.StepWrapper{ + { + Step: &ValidateCiliumMetric{ + KubeConfigFilePath: "./test.pem", + RetinaDaemonSetNamespace: common.KubeSystemNamespace, + RetinaDaemonSetName: "retina-agent-win", + }, + }, + } + return types.NewScenario(name, steps...) +} diff --git a/test/e2e/yaml/windows/install-ebpf-xdp.yaml b/test/e2e/yaml/windows/install-ebpf-xdp.yaml new file mode 100644 index 0000000000..eb7bd81b4f --- /dev/null +++ b/test/e2e/yaml/windows/install-ebpf-xdp.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: ebpf-xdp-install + namespace: ebpf-xdp-install # Ensure this namespace exists +spec: + selector: + matchLabels: + name: ebpf-xdp-install + template: + metadata: + labels: + name: ebpf-xdp-install + spec: + containers: + - name: ebpf-xdp-install-container + image: mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0 + command: + - powershell.exe + - -command + - 'echo "Installing eBPF XDP..."; Invoke-WebRequest -Uri "https://raw.githubusercontent.com/vpidatala94/retina/user/vpidatala/POC/8/scripts/install-ebpf-xdp.ps1" -OutFile "C:\\install-ebpf-xdp.ps1"; & C:\\install-ebpf-xdp.ps1; while ($true) { Start-Sleep -Seconds 300; }' + readinessProbe: + exec: + command: + - powershell.exe + - -command + - Test-Path C:\install-ebpf-xdp-probe-ready + initialDelaySeconds: 10 + periodSeconds: 5 + hostNetwork: true + nodeSelector: + kubernetes.io/os: windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" \ No newline at end of file diff --git a/test/e2e/yaml/windows/install-event-writer.yaml b/test/e2e/yaml/windows/install-event-writer.yaml new file mode 100644 index 0000000000..15ef4f074b --- /dev/null +++ b/test/e2e/yaml/windows/install-event-writer.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: install-event-writer + namespace: install-event-writer # Ensure this namespace exists +spec: + selector: + matchLabels: + name: install-event-writer + template: + metadata: + labels: + name: install-event-writer + spec: + containers: + - name: install-event-writer-container + image: mcr.microsoft.com/windows/servercore:ltsc2022 + command: + - powershell.exe + - -command + - 'echo "Installing event-writer..."; Invoke-WebRequest -Uri "https://github.com/vpidatala94/retina/raw/user/vpidatala/POC/8/test/plugin/eventwriter/x64/Release/bpf_event_writer.sys" -OutFile "C:\\bpf_event_writer.sys"; Invoke-WebRequest -Uri "https://github.com/vpidatala94/retina/raw/user/vpidatala/POC/8/test/plugin/eventwriter/x64/Release/event_writer.exe" -OutFile "C:\\event_writer.exe"; cd C:\\ ; .\event_writer.exe; New-Item -Path "C:\\install-event-writer-probe-ready" -ItemType File -Force; while ($true) { Start-Sleep -Seconds 300; }' + readinessProbe: + exec: + command: + - powershell.exe + - -command + - Test-Path C:\install-event-writer-probe-ready + initialDelaySeconds: 10 + periodSeconds: 5 + hostNetwork: true + nodeSelector: + kubernetes.io/os: windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" \ No newline at end of file diff --git a/test/image/Dockerfile b/test/image/Dockerfile index 8c681a3fee..1cd3ad2f63 100644 --- a/test/image/Dockerfile +++ b/test/image/Dockerfile @@ -2,7 +2,8 @@ # mcr.microsoft.com/oss/go/microsoft/golang:1.23.4-cbl-mariner2.0 FROM mcr.microsoft.com/oss/go/microsoft/golang@sha256:88225e171f29fe5f1f6ffca8eb659535b19b253354e43e1f4fc8a9bc67615ca1 AS builder ENV CGO_ENABLED=0 -COPY . /go/src/github.com/microsoft/retina +COPY . /go/src/github.com/microsoft/retina +RUN dir /go/src/github.com/microsoft/retina/pkg/plugin/ebpfwindows WORKDIR /go/src/github.com/microsoft/retina RUN tdnf install -y clang16 lld16 bpftool libbpf-devel make git RUN go generate /go/src/github.com/microsoft/retina/pkg/plugin/... diff --git a/pkg/plugin/ebpfwindows/event_writer/bpf_event_writer.c b/test/plugin/eventwriter/bpf_event_writer.c similarity index 100% rename from pkg/plugin/ebpfwindows/event_writer/bpf_event_writer.c rename to test/plugin/eventwriter/bpf_event_writer.c diff --git a/pkg/plugin/ebpfwindows/event_writer/event_writer_api.cpp b/test/plugin/eventwriter/event_writer.cpp similarity index 61% rename from pkg/plugin/ebpfwindows/event_writer/event_writer_api.cpp rename to test/plugin/eventwriter/event_writer.cpp index 303829c6d0..b55f3c7506 100644 --- a/pkg/plugin/ebpfwindows/event_writer/event_writer_api.cpp +++ b/test/plugin/eventwriter/event_writer.cpp @@ -1,14 +1,16 @@ +#include +#include #include #include -#include + #include #include "event_writer.h" -#include "event_writer_util.h" +#include std::vector> link_list; bpf_object* obj = NULL; -extern "C" __declspec(dllexport) DWORD +int set_filter(struct filter* flt) { uint8_t key = 0; int map_flt_fd = 0; @@ -26,24 +28,66 @@ set_filter(struct filter* flt) { return 0; } -extern "C" __declspec(dllexport) DWORD -check_five_tuple_exists(struct five_tuple* fvt) { - int map_evt_req_fd; - int value = 0; +int pin_map(const char* pin_path, bpf_map* map) { + int map_fd = 0; + // Attempt to open the pinned map + map_fd = bpf_obj_get(pin_path); + if (map_fd < 0) { + // Get the file descriptor of the map + map_fd = bpf_map__fd(map); + + if (map_fd < 0) { + fprintf(stderr, "%s - failed to get map file descriptor\n", __FUNCTION__); + return 1; + } - map_evt_req_fd = bpf_obj_get(FIVE_TUPLE_MAP_PIN_PATH); - if (map_evt_req_fd < 0) { - return 1; + if (bpf_obj_pin(map_fd, pin_path) < 0) { + fprintf(stderr, "%s - failed to pin map to %s\n", pin_path, __FUNCTION__); + return 1; + } + + printf("%s - map successfully pinned at %s\n", pin_path, __FUNCTION__); + } else { + printf("%s -pinned map found at %s\n", pin_path, __FUNCTION__); } - if (bpf_map_lookup_elem(map_evt_req_fd, fvt, &value) != 0) { - return 1; + return 0; +} + +std::vector get_physical_interface_indices() +{ + std::vector physical_indices; + ULONG flags = GAA_FLAG_INCLUDE_PREFIX; + ULONG family = AF_UNSPEC; + ULONG outBufLen = 0; + PIP_ADAPTER_ADDRESSES pAddresses = NULL; + PIP_ADAPTER_ADDRESSES pCurrAddresses = NULL; + + // Get the size needed for the buffer + if (GetAdaptersAddresses(family, flags, NULL, pAddresses, &outBufLen) == ERROR_BUFFER_OVERFLOW) { + pAddresses = (IP_ADAPTER_ADDRESSES*)malloc(outBufLen); + } + + // Get the actual data + if (GetAdaptersAddresses(family, flags, NULL, pAddresses, &outBufLen) == NO_ERROR) { + pCurrAddresses = pAddresses; + while (pCurrAddresses) { + if (pCurrAddresses->IfType == IF_TYPE_ETHERNET_CSMACD && pCurrAddresses->OperStatus == IfOperStatusUp) { + physical_indices.push_back(pCurrAddresses->IfIndex); + } + pCurrAddresses = pCurrAddresses->Next; + } } - return 0; + if (pAddresses) { + free(pAddresses); + } + + return physical_indices; } -extern "C" __declspec(dllexport) DWORD +int attach_program_to_interface(int ifindx) { + printf("%s - Attaching program to interface with ifindex %d\n", __FUNCTION__, ifindx); struct bpf_program* prg = bpf_object__find_program_by_name(obj, "event_writer"); bpf_link* link = NULL; if (prg == NULL) { @@ -61,7 +105,7 @@ attach_program_to_interface(int ifindx) { return 0; } -extern "C" __declspec(dllexport) DWORD +int pin_maps_load_programs(void) { struct bpf_program* prg = NULL; struct bpf_map *map_ev, *map_met, *map_fvt, *map_flt; @@ -133,7 +177,7 @@ pin_maps_load_programs(void) { } // Function to unload programs and detach -extern "C" __declspec(dllexport) DWORD +int unload_programs_detach() { for (auto it = link_list.begin(); it != link_list.end(); it ++) { auto ifidx = it->first; @@ -153,3 +197,25 @@ unload_programs_detach() { return 0; } + +int main(int argc, char* argv[]) { + int ret; + + printf ("Starting event writer\n"); + ret = pin_maps_load_programs(); + if (ret != 0) { + return ret; + } + std::vector interface_indices = get_physical_interface_indices(); + for (int ifindx : interface_indices) { + ret = attach_program_to_interface(ifindx); + if (ret != 0) { + return ret; + } + } + + //Sleep for 10 minutes + Sleep(600000); + unload_programs_detach(); + return 0; +} \ No newline at end of file diff --git a/pkg/plugin/ebpfwindows/event_writer/event_writer.h b/test/plugin/eventwriter/event_writer.h similarity index 100% rename from pkg/plugin/ebpfwindows/event_writer/event_writer.h rename to test/plugin/eventwriter/event_writer.h diff --git a/pkg/plugin/ebpfwindows/event_writer/event_writer.sln b/test/plugin/eventwriter/event_writer.sln similarity index 100% rename from pkg/plugin/ebpfwindows/event_writer/event_writer.sln rename to test/plugin/eventwriter/event_writer.sln diff --git a/pkg/plugin/ebpfwindows/event_writer/event_writer.vcxproj b/test/plugin/eventwriter/event_writer.vcxproj similarity index 69% rename from pkg/plugin/ebpfwindows/event_writer/event_writer.vcxproj rename to test/plugin/eventwriter/event_writer.vcxproj index 386c17b505..2197cf7402 100644 --- a/pkg/plugin/ebpfwindows/event_writer/event_writer.vcxproj +++ b/test/plugin/eventwriter/event_writer.vcxproj @@ -9,10 +9,6 @@ 10.0.26100.2454 - - Debug - x64 - Release x64 @@ -28,14 +24,8 @@ $(SolutionDir)packages\XDP-for-Windows.1.1.0\build\native\ - - Application - true - v143 - Unicode - - DynamicLibrary + Application false v143 true @@ -46,49 +36,22 @@ - - - - - event_writer - event_writer - - - Level3 - true - _DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) - true - $(SolutionDir)packages\eBPF-for-Windows.x64.0.20.0\build\native\include;$(SolutionDir)packages\XDP-for-Windows.1.1.0\build\native\include;%(AdditionalIncludeDirectories) - stdcpp20 - ProgramDatabase - - - Console - true - false - $(SolutionDir)packages\eBPF-for-Windows.x64.0.20.0\build\native\lib;$(SolutionDir)packages\XDP-for-Windows.1.1.0\build\native\lib;%(AdditionalLibraryDirectories) - ebpfapi.lib;%(AdditionalDependencies) - - - true - - Level3 true true true - NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + NDEBUG;_WINDOWS;%(PreprocessorDefinitions) true - $(SolutionDir)packages\eBPF-for-Windows.x64.0.20.0\build\native\include;$(SolutionDir)packages\XDP-for-Windows.1.1.0\build\native\include;%(AdditionalIncludeDirectories) + $(SolutionDir)packages\Microsoft.Windows.SDK.CPP.10.0.26100.2454\c\Include\10.0.26100.0\um;$(SolutionDir)packages\eBPF-for-Windows.x64.0.20.0\build\native\include;$(SolutionDir)packages\XDP-for-Windows.1.1.0\build\native\include;%(AdditionalIncludeDirectories) stdcpp20 @@ -97,21 +60,15 @@ true true false - $(SolutionDir)packages\eBPF-for-Windows.x64.0.20.0\build\native\lib;$(SolutionDir)packages\XDP-for-Windows.1.1.0\build\native\lib;%(AdditionalLibraryDirectories) - ebpfapi.lib;%(AdditionalDependencies) + $(SolutionDir)packages\Microsoft.Windows.SDK.CPP.x64.10.0.26100.2454\c\um\x64;$(SolutionDir)packages\eBPF-for-Windows.x64.0.20.0\build\native\lib;$(SolutionDir)packages\XDP-for-Windows.1.1.0\build\native\lib;%(AdditionalLibraryDirectories) + ebpfapi.lib;iphlpapi.lib;%(AdditionalDependencies) - - - - - - - + diff --git a/pkg/plugin/ebpfwindows/event_writer/event_writer.vcxproj.filters b/test/plugin/eventwriter/event_writer.vcxproj.filters similarity index 68% rename from pkg/plugin/ebpfwindows/event_writer/event_writer.vcxproj.filters rename to test/plugin/eventwriter/event_writer.vcxproj.filters index d6dfd78037..a433b610b7 100644 --- a/pkg/plugin/ebpfwindows/event_writer/event_writer.vcxproj.filters +++ b/test/plugin/eventwriter/event_writer.vcxproj.filters @@ -1,7 +1,8 @@  - + + @@ -14,4 +15,8 @@ + + + + \ No newline at end of file diff --git a/pkg/plugin/ebpfwindows/event_writer/event_writer.vcxproj.user b/test/plugin/eventwriter/event_writer.vcxproj.user similarity index 100% rename from pkg/plugin/ebpfwindows/event_writer/event_writer.vcxproj.user rename to test/plugin/eventwriter/event_writer.vcxproj.user diff --git a/test/plugin/eventwriter/event_writer/x64/Debug/event_writer.log b/test/plugin/eventwriter/event_writer/x64/Debug/event_writer.log new file mode 100644 index 0000000000..94f7fb7281 --- /dev/null +++ b/test/plugin/eventwriter/event_writer/x64/Debug/event_writer.log @@ -0,0 +1 @@ +C:\Users\vpidatala\Code\vinodRetina\retina\test\plugin\eventwriter\event_writer.vcxproj(140,5): error : This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is packages\eBPF-for-Windows.x64.0.20.0\build\native\ebpf-for-windows.x64.props. diff --git a/test/plugin/eventwriter/packages.config b/test/plugin/eventwriter/packages.config new file mode 100644 index 0000000000..39a3e211d6 --- /dev/null +++ b/test/plugin/eventwriter/packages.config @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/test/plugin/eventwriter/x64/Release/bpf_event_writer.sys b/test/plugin/eventwriter/x64/Release/bpf_event_writer.sys new file mode 100644 index 0000000000..08a74e2102 Binary files /dev/null and b/test/plugin/eventwriter/x64/Release/bpf_event_writer.sys differ diff --git a/test/plugin/eventwriter/x64/Release/event_writer.exe b/test/plugin/eventwriter/x64/Release/event_writer.exe new file mode 100644 index 0000000000..1565c885d5 Binary files /dev/null and b/test/plugin/eventwriter/x64/Release/event_writer.exe differ