[CI] Backport buildkite integration tests to 8.17 (#6323)

* [CI] Backport buildkite integration tests from main

* Remove devmachine target. A follow-up backport is required

* Remove devmachine target. A follow-up backport is required

* Remove devmachine target. A follow-up backport is required

* Remove devmachine target. A follow-up backport is required

* Remove devmachine target. A follow-up backport is required

* Update test report paths

* Integration tests: Parse agent version from version.go

Author: pazone

.buildkite/bk.integration.pipeline.yml

@@ -4,23 +4,12 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   ASDF_MAGE_VERSION: 1.14.0
-  ASDF_GOLANG_VERSION: 1.22.6
-  ASDF_TERRAFORM_VERSION: 1.9.3
 
 steps:
-  - label: "Integration tests: packaging"
-    key: "package-it"
-    command: ".buildkite/scripts/steps/integration-package.sh"
-    artifact_paths:
-      - build/distributions/**
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-
   - label: Start ESS stack for integration tests
     key: integration-ess
-    depends_on:
-      - package-it
+    env:
+      ASDF_TERRAFORM_VERSION: 1.9.2
     command: |
       #!/usr/bin/env bash
       set -euo pipefail
@@ -32,124 +21,145 @@ steps:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
 
-  - group: "Stateful IT (Sudo): Ubuntu"
-    key: integration-tests
+  - group: "Stateful: Windows"
+    key: integration-tests-win
     depends_on:
-      - package-it
       - integration-ess
     steps:
-      - label: "Default"
-        key: stateful-ubuntu-default-sudo
+      - label: "Win2022:sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "default" "^(TestAPMConfig|TestDiagnosticsOptionalValues|TestIsolatedUnitsDiagnosticsOptionalValues|TestDiagnosticsCommand|TestIsolatedUnitsDiagnosticsCommand|TestEventLogFile|TestFakeComponent|TestFakeIsolatedUnitsComponent|TestOtelFileProcessing|TestOtelLogsIngestion|TestOtelAPMIngestion|TestPackageVersion)$$"
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
+          - build/diagnostics/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-windows-2022"
+        matrix:
+          - default
+          - fleet
+          - fleet-privileged
+          - upgrade
 
-      - label: "Upgrade"
-        key: stateful-ubuntu-upgrade-sudo
+      - label: "Win2022:non-sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "upgrade" "^(TestUpgradeBrokenPackageVersion|TestStandaloneUpgradeWithGPGFallback|TestStandaloneUpgradeWithGPGFallbackOneRemoteFailing|TestStandaloneUpgradeRollback|TestStandaloneUpgradeRollbackOnRestarts|TestStandaloneUpgradeFailsWhenUpgradeIsInProgress|TestStandaloneUpgradeRetryDownload|TestStandaloneUpgradeSameCommit|TestStandaloneUpgrade|TestStandaloneUpgradeUninstallKillWatcher)$$"
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
+          - build/diagnostics/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-windows-2022"
+        matrix:
+          - default
 
-      - label: "Fleet"
-        key: stateful-ubuntu-fleet-sudo
+  - group: "Stateful:Ubuntu"
+    key: integration-tests-ubuntu
+    depends_on:
+      - integration-ess
+    steps:
+      - label: "x86_64:non-sudo: {{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet" "^(TestLongRunningAgentForLeaks|TestDelayEnroll|TestDelayEnrollUnprivileged|TestInstallAndCLIUninstallWithEndpointSecurity|TestInstallAndUnenrollWithEndpointSecurity|TestInstallWithEndpointSecurityAndRemoveEndpointIntegration|TestEndpointSecurityNonDefaultBasePath|TestEndpointSecurityUnprivileged|TestEndpointSecurityCannotSwitchToUnprivileged|TestEndpointLogsAreCollectedInDiagnostics|TestForceInstallOverProtectedPolicy|TestSetLogLevelFleetManaged|TestLogIngestionFleetManaged|TestMetricsMonitoringCorrectBinaries|TestEndpointAgentServiceMonitoring|TestMonitoringPreserveTextConfig|TestMonitoringLivenessReloadable|TestComponentBuildHashInDiagnostics|TestProxyURL|TestFleetManagedUpgradeUnprivileged)$$"
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
+          - build/diagnostics/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2404"
+        matrix:
+          - default
 
-      - label: "FQDN"
-        key: stateful-ubuntu-fqdn-sudo
+      - label: "x86_64:sudo: {{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fqdn" "^(TestFQDN)$$"
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
+          - build/diagnostics/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2404"
+        matrix:
+          - default
+          - container
+          - fleet-upgrade-to-pr-build
+          - upgrade
+          - fleet
+          - fqdn
+          - deb
+          - fleet-airgapped
+          - fleet-privileged
+          - fleet-airgapped-privileged
 
-      - label: "Deb"
-        key: stateful-ubuntu-deb-sudo
+      - label: "arm:sudo: {{matrix}}"
+        skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "deb" "^(TestDebLogIngestFleetManaged|TestDebFleetUpgrade)$$"
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
+          - build/diagnostics/**
         agents:
-          provider: "gcp"
-          imageProject: elastic-images-qa
-          machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          provider: "aws"
+          imagePrefix: "platform-ingest-beats-ubuntu-2404-aarch64"
+          instanceType: "m6g.2xlarge"
+        matrix:
+          - default
+          - container
+          - fleet-upgrade-to-pr-build
+          - upgrade
+          - fleet
+          - fqdn
+          - deb
+          - fleet-airgapped
+          - fleet-privileged
+          - fleet-airgapped-privileged
 
-      - label: "Fleet Airgapped"
-        key: stateful-ubuntu-fleet-airgapped-sudo
+      - label: "arm:non-sudo: {{matrix}}"
+        skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped" "^(TestFleetAirGappedUpgradeUnprivileged)$$"
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
+          - build/diagnostics/**
         agents:
-          provider: "gcp"
-          imageProject: elastic-images-qa
-          machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          provider: "aws"
+          imagePrefix: "platform-ingest-beats-ubuntu-2404-aarch64"
+          instanceType: "m6g.xlarge"
+        matrix:
+          - default
 
-      - label: "Fleet Privileged"
-        key: stateful-ubuntu-fleet-privileged-sudo
-        command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-privileged" "^(TestInstallFleetServerBootstrap|TestFleetManagedUpgradePrivileged)$$"
-        artifact_paths:
-          - build/**
-        agents:
-          provider: "gcp"
-          imageProject: elastic-images-qa
-          machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
-
-      - label: "Fleet Airgapped Privileged"
-        key: stateful-ubuntu-fleet-airgapped-privileged-sudo
+  - group: "Stateful(Sudo):RHEL8"
+    key: integration-tests-rhel8
+    depends_on:
+      - integration-ess
+    steps:
+      - label: "x86_64:sudo:rpm"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped-privileged" "^(TestFleetAirGappedUpgradePrivileged)$$"
+          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**
+          - build/diagnostics/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-rhel-8"
 
   - label: ESS stack cleanup
-    depends_on: integration-tests
+    depends_on:
+      - integration-tests-ubuntu
+      - integration-tests-win
+      - integration-tests-rhel8
     allow_dependency_failure: true
     command: |
       buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
@@ -158,3 +168,26 @@ steps:
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
+
+  - label: Aggregate test reports
+    # Warning: The key has a hook in pre-command
+    key: aggregate-reports
+    depends_on:
+      - integration-tests-ubuntu
+      - integration-tests-win
+      - integration-tests-rhel8
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "build/*.xml" .
+      buildkite-agent artifact download "build\*.xml" .
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+    soft_fail:
+      - exit_status: "*"
+    plugins:
+      - test-collector#v1.10.1:
+          files: "build/*.xml"
+          format: "junit"
+          branches: "main"
+          debug: true

.buildkite/hooks/pre-command

@@ -63,7 +63,13 @@ if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   echo "--- Prepare BK test analytics token :vault:"
   BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
   export BUILDKITE_ANALYTICS_TOKEN
+fi
 
+if [[ "$BUILDKITE_STEP_KEY" == *"aggregate-reports"* ]]; then
+  # BK analytics
+  echo "--- Prepare BK test analytics token :vault:"
+  BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
+  export BUILDKITE_ANALYTICS_TOKEN
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then

.buildkite/hooks/pre-command.ps1

@@ -1,32 +1,2 @@
 # Shorten BUILDKITE_MESSAGE if needed to avoid filling the Windows env var buffer
 $env:BUILDKITE_MESSAGE = $env:BUILDKITE_MESSAGE.Substring(0, [System.Math]::Min(2048, $env:BUILDKITE_MESSAGE.Length))
-
-# Install gcc TODO: Move to the VM image
-choco install mingw
-Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
-refreshenv
-
-# Install gvm and go
-# TODO: Move GVM download to the base VM image
-$env:GvmVersion = "0.5.2"
-[Net.ServicePointManager]::SecurityProtocol = "tls12"
-$env:GoVersion = Get-Content -Path .go-version
-Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
-gvm --format=powershell $env:GoVersion | Invoke-Expression
-go version
-
-$GOPATH = $(go env GOPATH)
-$env:Path = "$GOPATH\bin;" + $env:Path
-$env:GOTMPDIR = "$GOPATH\tmp"
-New-Item -ItemType Directory -Force -Path $env:GOTMPDIR
-[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
-[Environment]::SetEnvironmentVariable("GOTMPDIR", "$GOPATH\tmp", [EnvironmentVariableTarget]::Machine)
-[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
-go env
-
-# Install tools
-go install github.com/magefile/mage
-go install github.com/elastic/go-licenser
-go install golang.org/x/tools/cmd/goimports
-go install github.com/jstemmer/go-junit-report
-go install gotest.tools/gotestsum

.buildkite/integration.pipeline.yml

@@ -57,32 +57,13 @@ steps:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
 
-  - label: "Integration tests"
-    key: "integration-tests"
+  - label: "Triggering Integration tests"
     depends_on:
       - package-it
-    concurrency_group: elastic-agent-extended-testing/integration
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f"
-    command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/integration_tests.sh stateful
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Integration tests"
-    plugins:
-      - test-collector#v1.10.1:
-          files: "build/TEST-*.xml"
-          format: "junit"
-          branches: "main"
-          debug: true
+    trigger: "elastic-agent-extended-testing-bk"
+    build:
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
 
   - label: "Serverless Beats Tests"
     depends_on: