Skip to content

Commit 03fef6c

Browse files
oakrizanmergify[bot]
oakrizan
authored and
mergify[bot]
committed
Split packaging into separate steps (#6401)
Splitting centralized packaging step into separate steps to speed up builds (cherry picked from commit e956b4d) # Conflicts: # .buildkite/scripts/steps/integration-package.sh
1 parent 8bc7c7b commit 03fef6c

File tree

3 files changed

+88
-25
lines changed

3 files changed

+88
-25
lines changed

‎.buildkite/bk.integration.pipeline.yml

+9-7
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ steps:
2828
steps:
2929
- label: "Win2022:sudo:{{matrix}}"
3030
command: |
31-
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
31+
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
3232
.buildkite/scripts/integration-tests.ps1 {{matrix}} true
3333
artifact_paths:
3434
- build/**
@@ -45,7 +45,7 @@ steps:
4545

4646
- label: "Win2022:non-sudo:{{matrix}}"
4747
command: |
48-
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
48+
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
4949
.buildkite/scripts/integration-tests.ps1 {{matrix}} false
5050
artifact_paths:
5151
- build/**
@@ -63,8 +63,9 @@ steps:
6363
- integration-ess
6464
steps:
6565
- label: "x86_64:non-sudo: {{matrix}}"
66+
# only packaging-ubuntu-x86-64 artifact dependency is required
6667
command: |
67-
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
68+
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
6869
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
6970
artifact_paths:
7071
- build/**
@@ -77,8 +78,9 @@ steps:
7778
- default
7879

7980
- label: "x86_64:sudo: {{matrix}}"
81+
# due to deb group present in matrix tar.gz and deb packages artifacts are required
8082
command: |
81-
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
83+
buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
8284
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
8385
artifact_paths:
8486
- build/**
@@ -102,7 +104,7 @@ steps:
102104
- label: "arm:sudo: {{matrix}}"
103105
skip: true
104106
command: |
105-
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
107+
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
106108
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
107109
artifact_paths:
108110
- build/**
@@ -126,7 +128,7 @@ steps:
126128
- label: "arm:non-sudo: {{matrix}}"
127129
skip: true
128130
command: |
129-
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
131+
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
130132
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
131133
artifact_paths:
132134
- build/**
@@ -145,7 +147,7 @@ steps:
145147
steps:
146148
- label: "x86_64:sudo:rpm"
147149
command: |
148-
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
150+
buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
149151
.buildkite/scripts/steps/integration_tests_tf.sh rpm true
150152
artifact_paths:
151153
- build/**

‎.buildkite/integration.pipeline.yml

+69-18
Original file line numberDiff line numberDiff line change
@@ -5,80 +5,131 @@ env:
55
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
66

77
steps:
8-
- label: "Integration tests: packaging"
9-
key: "package-it"
10-
command: ".buildkite/scripts/steps/integration-package.sh"
11-
artifact_paths:
12-
- build/distributions/**
13-
agents:
14-
provider: "gcp"
15-
machineType: "n1-standard-8"
8+
- group: "Integration tests: packaging"
9+
key: "int-packaging"
10+
steps:
11+
- label: "Packaging: Ubuntu x86_64"
12+
key: "packaging-ubuntu-x86-64"
13+
env:
14+
PACKAGES: "tar.gz"
15+
PLATFORMS: "linux/amd64"
16+
command: ".buildkite/scripts/steps/integration-package.sh"
17+
artifact_paths:
18+
- build/distributions/**
19+
agents:
20+
provider: "gcp"
21+
machineType: "n2-standard-8"
22+
23+
- label: "Packaging: Ubuntu arm64"
24+
key: "packaging-ubuntu-arm64"
25+
env:
26+
PACKAGES: "tar.gz"
27+
PLATFORMS: "linux/arm64"
28+
command: ".buildkite/scripts/steps/integration-package.sh"
29+
artifact_paths:
30+
- build/distributions/**
31+
agents:
32+
provider: "gcp"
33+
machineType: "n2-standard-8"
34+
35+
- label: "Packaging: Windows"
36+
key: "packaging-windows"
37+
env:
38+
PACKAGES: "zip"
39+
PLATFORMS: "windows/amd64"
40+
command: ".buildkite/scripts/steps/integration-package.sh"
41+
artifact_paths:
42+
- build/distributions/**
43+
agents:
44+
provider: "gcp"
45+
machineType: "n2-standard-8"
46+
47+
- label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}"
48+
key: "packaging-containers"
49+
env:
50+
PACKAGES: "{{matrix.ext}}"
51+
PLATFORMS: "{{matrix.arch}}"
52+
command: ".buildkite/scripts/steps/integration-package.sh"
53+
artifact_paths:
54+
- build/distributions/**
55+
agents:
56+
provider: "gcp"
57+
machineType: "n2-standard-8"
58+
matrix:
59+
setup:
60+
arch:
61+
- linux/amd64
62+
- linux/arm64
63+
ext:
64+
- rpm
65+
- deb
1666

1767
- label: "Serverless integration test"
1868
key: "serverless-integration-tests"
1969
depends_on:
20-
- package-it
70+
- int-packaging
2171
concurrency_group: elastic-agent-extended-testing/serverless-integration
2272
concurrency: 8
2373
env:
2474
# we run each step in a different data center to spread the load
2575
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
2676
command: |
27-
buildkite-agent artifact download build/distributions/** . --step 'package-it'
77+
buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
2878
.buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
2979
artifact_paths:
3080
- "build/TEST-**"
3181
- "build/diagnostics/*"
3282
agents:
3383
provider: "gcp"
34-
machineType: "n1-standard-8"
84+
machineType: "n2-standard-8"
3585
notify:
3686
- github_commit_status:
3787
context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
3888

3989
- label: "Extended runtime leak tests"
4090
key: "extended-integration-tests"
4191
depends_on:
42-
- package-it
92+
- int-packaging
4393
concurrency_group: elastic-agent-extended-testing/leak-tests
4494
concurrency: 8
4595
env:
4696
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
4797
command: |
48-
buildkite-agent artifact download build/distributions/** . --step 'package-it'
98+
buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
4999
.buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
50100
artifact_paths:
51101
- "build/TEST-**"
52102
- "build/diagnostics/*"
53103
agents:
54104
provider: "gcp"
55-
machineType: "n1-standard-8"
105+
machineType: "n2-standard-8"
56106
notify:
57107
- github_commit_status:
58108
context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
59109

60110
- label: "Triggering Integration tests"
61111
depends_on:
62-
- package-it
112+
- int-packaging
63113
trigger: "elastic-agent-extended-testing-bk"
64114
build:
65115
commit: "${BUILDKITE_COMMIT}"
66116
branch: "${BUILDKITE_BRANCH}"
67117

68118
- label: "Serverless Beats Tests"
119+
# To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required
69120
depends_on:
70-
- package-it
121+
- packaging-ubuntu-x86-64
71122
key: "serverless-beats-integration-tests"
72123
concurrency_group: elastic-agent-extended-testing/beats-integration
73124
concurrency: 8
74125
env:
75126
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
76127
command: |
77-
buildkite-agent artifact download build/distributions/** . --step 'package-it'
128+
buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
78129
.buildkite/scripts/steps/beats_tests.sh
79130
agents:
80131
provider: "gcp"
81-
machineType: "n1-standard-8"
132+
machineType: "n2-standard-8"
82133
notify:
83134
- github_commit_status:
84135
context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"

‎.buildkite/scripts/steps/integration-package.sh

+10
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,14 @@ set -euo pipefail
33

44
source .buildkite/scripts/common.sh
55

6+
<<<<<<< HEAD
67
PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package
8+
=======
9+
# Remove AGENT_PACKAGE_VERSION pinning as soon as 9.0.0 is released
10+
export AGENT_PACKAGE_VERSION=9.0.0
11+
12+
export SNAPSHOT="true"
13+
export EXTERNAL="true"
14+
15+
mage package
16+
>>>>>>> e956b4d2d (Split packaging into separate steps (#6401))

0 commit comments

Comments
 (0)