Skip to content

Commit

Permalink
[POC] Flaky test retries
Browse files Browse the repository at this point in the history
  • Loading branch information
smola committed Dec 18, 2024
1 parent a19f73a commit 181cdef
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 74 deletions.
72 changes: 1 addition & 71 deletions .circleci/config.continue.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -1122,7 +1122,6 @@ build_test_jobs: &build_test_jobs
name: z_test_<< matrix.testJvm >>_base
triggeredBy: *core_modules
gradleTarget: ":baseTest"
gradleParameters: "-PskipFlakyTests"
stage: core
cacheType: base
parallelism: 4
Expand All @@ -1136,7 +1135,7 @@ build_test_jobs: &build_test_jobs
name: z_test_8_base
triggeredBy: *core_modules
gradleTarget: :baseTest
gradleParameters: "-PskipFlakyTests -PcheckCoverage"
gradleParameters: "-PcheckCoverage"
stage: core
cacheType: base
parallelism: 4
Expand All @@ -1148,7 +1147,6 @@ build_test_jobs: &build_test_jobs
- ok_to_test
name: z_test_<< matrix.testJvm >>_inst
gradleTarget: ":instrumentationTest"
gradleParameters: "-PskipFlakyTests"
triggeredBy: *instrumentation_modules
stage: instrumentation
cacheType: inst
Expand All @@ -1162,7 +1160,6 @@ build_test_jobs: &build_test_jobs
- ok_to_test
name: z_test_8_inst
gradleTarget: ":instrumentationTest"
gradleParameters: "-PskipFlakyTests"
triggeredBy: *instrumentation_modules
stage: instrumentation
cacheType: inst
Expand All @@ -1176,7 +1173,6 @@ build_test_jobs: &build_test_jobs
- build_latestdep
name: test_8_inst_latest
gradleTarget: ":instrumentationLatestDepTest"
gradleParameters: "-PskipFlakyTests"
triggeredBy: *instrumentation_modules
stage: instrumentation
cacheType: latestdep
Expand All @@ -1190,7 +1186,6 @@ build_test_jobs: &build_test_jobs
- build_latestdep
name: test_17_inst_latest
gradleTarget: ":instrumentationLatestDepTest"
gradleParameters: "-PskipFlakyTests"
triggeredBy: *instrumentation_modules
stage: instrumentation
cacheType: latestdep
Expand All @@ -1204,77 +1199,18 @@ build_test_jobs: &build_test_jobs
- build_latestdep
name: test_21_inst_latest
gradleTarget: ":instrumentationLatestDepTest"
gradleParameters: "-PskipFlakyTests"
triggeredBy: *instrumentation_modules
stage: instrumentation
cacheType: latestdep
parallelism: 12
maxWorkers: 3
testJvm: "21"
{% if flaky %}
- tests:
requires:
- ok_to_test
name: z_test_8_flaky_base
gradleTarget: ":baseTest"
gradleParameters: "-PrunFlakyTests"
continueOnFailure: true
triggeredBy: *core_modules
stage: core
cacheType: base
parallelism: 4
maxWorkers: 4
testJvm: "8"
- xlarge_tests:
requires:
- ok_to_test
name: z_test_8_flaky_inst
gradleTarget: ":instrumentationTest"
gradleParameters: "-PrunFlakyTests"
continueOnFailure: true
triggeredBy: *instrumentation_modules
stage: instrumentation
cacheType: inst
parallelism: 12
maxWorkers: 4
testJvm: "8"
- tests:
requires:
- ok_to_test
name: z_test_8_flaky_smoke
gradleTarget: ":smokeTest"
gradleParameters: "-PrunFlakyTests"
continueOnFailure: true
stage: smoke
cacheType: smoke
parallelism: 4
maxWorkers: 4
testJvm: "8"
- tests:
requires:
- ok_to_test
name: z_test_8_flaky_debugger
gradleTarget: ":debuggerTest"
gradleParameters: "-PrunFlakyTests"
continueOnFailure: true
triggeredBy: *debugger_modules
stage: debugger
cacheType: base
parallelism: 4
maxWorkers: 4
testJvm: "8"
{% endif %}
- tests:
requires:
- ok_to_test
maxWorkers: 4
gradleTarget: ":profilingTest"
gradleParameters: "-PskipFlakyTests"
triggeredBy: *profiling_modules
stage: profiling
cacheType: profiling
Expand All @@ -1288,7 +1224,6 @@ build_test_jobs: &build_test_jobs
name: test_<< matrix.testJvm >>_debugger
maxWorkers: 4
gradleTarget: ":debuggerTest"
gradleParameters: "-PskipFlakyTests"
triggeredBy: *debugger_modules
stage: debugger
cacheType: base
Expand All @@ -1300,7 +1235,6 @@ build_test_jobs: &build_test_jobs
- ok_to_test
name: z_test_<< matrix.testJvm >>_smoke
gradleTarget: "stageMainDist :smokeTest"
gradleParameters: "-PskipFlakyTests"
stage: smoke
cacheType: smoke
parallelism: 4
Expand All @@ -1314,7 +1248,6 @@ build_test_jobs: &build_test_jobs
name: z_test_<< matrix.testJvm >>_ssi_smoke
environment: "DD_INJECT_FORCE=true DD_INJECTION_ENABLED=tracer"
gradleTarget: "stageMainDist :smokeTest"
gradleParameters: "-PskipFlakyTests"
stage: smoke
cacheType: smoke
parallelism: 4
Expand All @@ -1328,7 +1261,6 @@ build_test_jobs: &build_test_jobs
name: test_semeru8_debugger_smoke
maxWorkers: 4
gradleTarget: "stageMainDist dd-smoke-tests:debugger-integration-tests:test"
gradleParameters: "-PskipFlakyTests"
triggeredBy: *debugger_modules
stage: debugger
cacheType: smoke
Expand Down Expand Up @@ -1357,7 +1289,6 @@ build_test_jobs: &build_test_jobs
- ok_to_test
name: z_test_8_smoke
gradleTarget: "stageMainDist :smokeTest"
gradleParameters: "-PskipFlakyTests"
stage: smoke
cacheType: smoke
parallelism: 4
Expand All @@ -1370,7 +1301,6 @@ build_test_jobs: &build_test_jobs
name: z_test_8_ssi_smoke
environment: "DD_INJECT_FORCE=true DD_INJECTION_ENABLED=tracer"
gradleTarget: "stageMainDist :smokeTest"
gradleParameters: "-PskipFlakyTests"
stage: smoke
cacheType: smoke
parallelism: 4
Expand Down
17 changes: 15 additions & 2 deletions gradle/configure_tests.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,21 @@ tasks.withType(Test).configureEach {

Task allTestsTask = tasks.maybeCreate('allTests')
Task allLatestDepTestsTask = tasks.maybeCreate('allLatestDepTests')
project.afterEvaluate {
tasks.withType(Test).each {
project.afterEvaluate { proj ->
if (System.getenv("CI") != null) {
proj.tasks.withType(Test).configureEach { task ->
task.extensions.configure('develocity') { it ->
it.testRetry.maxRetries = 3
it.testRetry.maxFailures = 20
it.testRetry.failOnPassedAfterRetry = false
it.testRetry.failOnSkippedAfterRetry = true
it.testRetry.filter { f ->
f.includeAnnotationClasses.add('datadog.trace.test.util.Flaky')
}
}
}
}
proj.tasks.withType(Test).each {
if (it.name.containsIgnoreCase('latest')) {
allLatestDepTestsTask.dependsOn it
} else if (it.name != 'traceAgentTest') {
Expand Down
2 changes: 1 addition & 1 deletion settings.gradle
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
plugins {
id 'com.gradle.develocity' version '3.18'
id 'com.gradle.develocity' version '3.19'
}

def isCI = System.getenv("CI") != null
Expand Down

0 comments on commit 181cdef

Please sign in to comment.