Skip to content

Commit e5426d8

Browse files
smolasmola
committed
[POC] Flaky test retries
1 parent a3e9bda commit e5426d8

File tree

3 files changed

+17
-74
lines changed

3 files changed

+17
-74
lines changed

Diff for: ‎.circleci/config.continue.yml.j2

+1-71
Original file line numberDiff line numberDiff line change
@@ -1122,7 +1122,6 @@ build_test_jobs: &build_test_jobs
11221122
name: z_test_<< matrix.testJvm >>_base
11231123
triggeredBy: *core_modules
11241124
gradleTarget: ":baseTest"
1125-
gradleParameters: "-PskipFlakyTests"
11261125
stage: core
11271126
cacheType: base
11281127
parallelism: 4
@@ -1136,7 +1135,7 @@ build_test_jobs: &build_test_jobs
11361135
name: z_test_8_base
11371136
triggeredBy: *core_modules
11381137
gradleTarget: :baseTest
1139-
gradleParameters: "-PskipFlakyTests -PcheckCoverage"
1138+
gradleParameters: "-PcheckCoverage"
11401139
stage: core
11411140
cacheType: base
11421141
parallelism: 4
@@ -1148,7 +1147,6 @@ build_test_jobs: &build_test_jobs
11481147
- ok_to_test
11491148
name: z_test_<< matrix.testJvm >>_inst
11501149
gradleTarget: ":instrumentationTest"
1151-
gradleParameters: "-PskipFlakyTests"
11521150
triggeredBy: *instrumentation_modules
11531151
stage: instrumentation
11541152
cacheType: inst
@@ -1162,7 +1160,6 @@ build_test_jobs: &build_test_jobs
11621160
- ok_to_test
11631161
name: z_test_8_inst
11641162
gradleTarget: ":instrumentationTest"
1165-
gradleParameters: "-PskipFlakyTests"
11661163
triggeredBy: *instrumentation_modules
11671164
stage: instrumentation
11681165
cacheType: inst
@@ -1176,7 +1173,6 @@ build_test_jobs: &build_test_jobs
11761173
- build_latestdep
11771174
name: test_8_inst_latest
11781175
gradleTarget: ":instrumentationLatestDepTest"
1179-
gradleParameters: "-PskipFlakyTests"
11801176
triggeredBy: *instrumentation_modules
11811177
stage: instrumentation
11821178
cacheType: latestdep
@@ -1190,7 +1186,6 @@ build_test_jobs: &build_test_jobs
11901186
- build_latestdep
11911187
name: test_17_inst_latest
11921188
gradleTarget: ":instrumentationLatestDepTest"
1193-
gradleParameters: "-PskipFlakyTests"
11941189
triggeredBy: *instrumentation_modules
11951190
stage: instrumentation
11961191
cacheType: latestdep
@@ -1204,77 +1199,18 @@ build_test_jobs: &build_test_jobs
12041199
- build_latestdep
12051200
name: test_21_inst_latest
12061201
gradleTarget: ":instrumentationLatestDepTest"
1207-
gradleParameters: "-PskipFlakyTests"
12081202
triggeredBy: *instrumentation_modules
12091203
stage: instrumentation
12101204
cacheType: latestdep
12111205
parallelism: 12
12121206
maxWorkers: 3
12131207
testJvm: "21"
12141208
1215-
{% if flaky %}
1216-
- tests:
1217-
requires:
1218-
- ok_to_test
1219-
name: z_test_8_flaky_base
1220-
gradleTarget: ":baseTest"
1221-
gradleParameters: "-PrunFlakyTests"
1222-
continueOnFailure: true
1223-
triggeredBy: *core_modules
1224-
stage: core
1225-
cacheType: base
1226-
parallelism: 4
1227-
maxWorkers: 4
1228-
testJvm: "8"
1229-
1230-
- xlarge_tests:
1231-
requires:
1232-
- ok_to_test
1233-
name: z_test_8_flaky_inst
1234-
gradleTarget: ":instrumentationTest"
1235-
gradleParameters: "-PrunFlakyTests"
1236-
continueOnFailure: true
1237-
triggeredBy: *instrumentation_modules
1238-
stage: instrumentation
1239-
cacheType: inst
1240-
parallelism: 12
1241-
maxWorkers: 4
1242-
testJvm: "8"
1243-
1244-
- tests:
1245-
requires:
1246-
- ok_to_test
1247-
name: z_test_8_flaky_smoke
1248-
gradleTarget: ":smokeTest"
1249-
gradleParameters: "-PrunFlakyTests"
1250-
continueOnFailure: true
1251-
stage: smoke
1252-
cacheType: smoke
1253-
parallelism: 4
1254-
maxWorkers: 4
1255-
testJvm: "8"
1256-
1257-
- tests:
1258-
requires:
1259-
- ok_to_test
1260-
name: z_test_8_flaky_debugger
1261-
gradleTarget: ":debuggerTest"
1262-
gradleParameters: "-PrunFlakyTests"
1263-
continueOnFailure: true
1264-
triggeredBy: *debugger_modules
1265-
stage: debugger
1266-
cacheType: base
1267-
parallelism: 4
1268-
maxWorkers: 4
1269-
testJvm: "8"
1270-
{% endif %}
1271-
12721209
- tests:
12731210
requires:
12741211
- ok_to_test
12751212
maxWorkers: 4
12761213
gradleTarget: ":profilingTest"
1277-
gradleParameters: "-PskipFlakyTests"
12781214
triggeredBy: *profiling_modules
12791215
stage: profiling
12801216
cacheType: profiling
@@ -1288,7 +1224,6 @@ build_test_jobs: &build_test_jobs
12881224
name: test_<< matrix.testJvm >>_debugger
12891225
maxWorkers: 4
12901226
gradleTarget: ":debuggerTest"
1291-
gradleParameters: "-PskipFlakyTests"
12921227
triggeredBy: *debugger_modules
12931228
stage: debugger
12941229
cacheType: base
@@ -1300,7 +1235,6 @@ build_test_jobs: &build_test_jobs
13001235
- ok_to_test
13011236
name: z_test_<< matrix.testJvm >>_smoke
13021237
gradleTarget: "stageMainDist :smokeTest"
1303-
gradleParameters: "-PskipFlakyTests"
13041238
stage: smoke
13051239
cacheType: smoke
13061240
parallelism: 4
@@ -1314,7 +1248,6 @@ build_test_jobs: &build_test_jobs
13141248
name: z_test_<< matrix.testJvm >>_ssi_smoke
13151249
environment: "DD_INJECT_FORCE=true DD_INJECTION_ENABLED=tracer"
13161250
gradleTarget: "stageMainDist :smokeTest"
1317-
gradleParameters: "-PskipFlakyTests"
13181251
stage: smoke
13191252
cacheType: smoke
13201253
parallelism: 4
@@ -1328,7 +1261,6 @@ build_test_jobs: &build_test_jobs
13281261
name: test_semeru8_debugger_smoke
13291262
maxWorkers: 4
13301263
gradleTarget: "stageMainDist dd-smoke-tests:debugger-integration-tests:test"
1331-
gradleParameters: "-PskipFlakyTests"
13321264
triggeredBy: *debugger_modules
13331265
stage: debugger
13341266
cacheType: smoke
@@ -1357,7 +1289,6 @@ build_test_jobs: &build_test_jobs
13571289
- ok_to_test
13581290
name: z_test_8_smoke
13591291
gradleTarget: "stageMainDist :smokeTest"
1360-
gradleParameters: "-PskipFlakyTests"
13611292
stage: smoke
13621293
cacheType: smoke
13631294
parallelism: 4
@@ -1370,7 +1301,6 @@ build_test_jobs: &build_test_jobs
13701301
name: z_test_8_ssi_smoke
13711302
environment: "DD_INJECT_FORCE=true DD_INJECTION_ENABLED=tracer"
13721303
gradleTarget: "stageMainDist :smokeTest"
1373-
gradleParameters: "-PskipFlakyTests"
13741304
stage: smoke
13751305
cacheType: smoke
13761306
parallelism: 4

Diff for: ‎gradle/configure_tests.gradle

+15-2
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,21 @@ tasks.withType(Test).configureEach {
8484

8585
Task allTestsTask = tasks.maybeCreate('allTests')
8686
Task allLatestDepTestsTask = tasks.maybeCreate('allLatestDepTests')
87-
project.afterEvaluate {
88-
tasks.withType(Test).each {
87+
project.afterEvaluate { proj ->
88+
if (System.getenv("CI") != null) {
89+
proj.tasks.withType(Test).configureEach { task ->
90+
task.extensions.configure('develocity') { it ->
91+
it.testRetry.maxRetries = 3
92+
it.testRetry.maxFailures = 20
93+
it.testRetry.failOnPassedAfterRetry = false
94+
it.testRetry.failOnSkippedAfterRetry = true
95+
it.testRetry.filter { f ->
96+
f.includeAnnotationClasses.add('datadog.trace.test.util.Flaky')
97+
}
98+
}
99+
}
100+
}
101+
proj.tasks.withType(Test).each {
89102
if (it.name.containsIgnoreCase('latest')) {
90103
allLatestDepTestsTask.dependsOn it
91104
} else if (it.name != 'traceAgentTest') {

Diff for: ‎settings.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
plugins {
2-
id 'com.gradle.develocity' version '3.18'
2+
id 'com.gradle.develocity' version '3.19'
33
}
44

55
def isCI = System.getenv("CI") != null

0 commit comments

Comments
 (0)