Implement fail-fast tests ordering for JUnit 5 (#8055)

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java

@@ -15,6 +15,7 @@
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.civisibility.telemetry.tag.CoverageEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionEnabled;
+import datadog.trace.api.civisibility.telemetry.tag.FlakyTestRetriesEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.ItrEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.ItrSkipEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.RequireGit;
@@ -132,6 +133,7 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) thr
         settings.getEarlyFlakeDetectionSettings().isEnabled()
             ? EarlyFlakeDetectionEnabled.TRUE
             : null,
+        settings.isFlakyTestRetriesEnabled() ? FlakyTestRetriesEnabled.TRUE : null,
         settings.isGitUploadRequired() ? RequireGit.TRUE : null);
 
     return settings;
@@ -185,6 +187,14 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) thr
   @Override
   public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
       TracerEnvironment tracerEnvironment) throws IOException {
+    OkHttpUtils.CustomListener telemetryListener =
+        new TelemetryListener.Builder(metricCollector)
+            .requestCount(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST)
+            .requestErrors(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST_ERRORS)
+            .requestDuration(CiVisibilityDistributionMetric.FLAKY_TESTS_REQUEST_MS)
+            .responseBytes(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_BYTES)
+            .build();
+
     String uuid = uuidGenerator.get();
     EnvelopeDto<TracerEnvironment> request =
         new EnvelopeDto<>(
@@ -196,11 +206,12 @@ public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
             FLAKY_TESTS_URI,
             requestBody,
             is -> testIdentifiersResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data,
-            null,
+            telemetryListener,
             false);
 
     LOGGER.debug("Received {} flaky tests in total", response.size());
 
+    int flakyTestsCount = 0;
     Map<String, Collection<TestIdentifier>> testIdentifiers = new HashMap<>();
     for (DataDto<TestIdentifierJson> dataDto : response) {
       TestIdentifierJson testIdentifierJson = dataDto.getAttributes();
@@ -209,7 +220,10 @@ public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
       testIdentifiers
           .computeIfAbsent(moduleName, k -> new HashSet<>())
           .add(testIdentifierJson.toTestIdentifier());
+      flakyTestsCount++;
     }
+
+    metricCollector.add(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_TESTS, flakyTestsCount);
     return testIdentifiers;
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java

@@ -114,7 +114,6 @@ public Collection<TestIdentifier> getKnownTests() {
 
   @Nullable
   public Collection<TestIdentifier> getFlakyTests() {
-    // backend does not store module info for flaky tests yet
     return flakyTests;
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java

@@ -1,6 +1,7 @@
 package datadog.trace.civisibility.config;
 
 import datadog.trace.api.Config;
+import datadog.trace.api.civisibility.CIConstants;
 import datadog.trace.api.civisibility.CiVisibilityWellKnownTags;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.config.TestMetadata;
@@ -139,11 +140,17 @@ private TracerEnvironment buildTracerEnvironment(
 
     Map<String, Collection<TestIdentifier>> flakyTestsByModule =
         flakyTestRetriesEnabled && config.isCiVisibilityFlakyRetryOnlyKnownFlakes()
+                || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(
+                    config.getCiVisibilityTestOrder())
             ? getFlakyTestsByModule(tracerEnvironment)
             : null;
 
     Map<String, Collection<TestIdentifier>> knownTestsByModule =
-        earlyFlakeDetectionEnabled ? getKnownTestsByModule(tracerEnvironment) : null;
+        earlyFlakeDetectionEnabled
+                || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(
+                    config.getCiVisibilityTestOrder())
+            ? getKnownTestsByModule(tracerEnvironment)
+            : null;
 
     Set<String> moduleNames = new HashSet<>(Collections.singleton(DEFAULT_SETTINGS));
     moduleNames.addAll(skippableTestIdentifiers.keySet());

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java

@@ -13,8 +13,10 @@ public static void serialize(Serializer serializer, TestIdentifier testIdentifie
   }
 
   public static TestIdentifier deserialize(ByteBuffer buffer) {
+    String suiteName = Serializer.readString(buffer);
     return new TestIdentifier(
-        Serializer.readString(buffer),
+        // suite name repeats a lot; interning it to save memory
+        suiteName != null ? suiteName.intern() : null,
         Serializer.readString(buffer),
         Serializer.readString(buffer));
   }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java

@@ -5,11 +5,14 @@
 import datadog.trace.api.Config;
 import datadog.trace.api.DDTraceId;
 import datadog.trace.api.IdGenerationStrategy;
+import datadog.trace.api.civisibility.CIConstants;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.civisibility.telemetry.TagValue;
+import datadog.trace.api.civisibility.telemetry.tag.AgentlessLogSubmissionEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.AutoInjected;
 import datadog.trace.api.civisibility.telemetry.tag.EventType;
+import datadog.trace.api.civisibility.telemetry.tag.FailFastTestOrderEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner;
 import datadog.trace.api.civisibility.telemetry.tag.IsHeadless;
 import datadog.trace.api.civisibility.telemetry.tag.IsUnsupportedCI;
@@ -109,7 +112,11 @@ public AbstractTestSession(
         CiVisibilityCountMetric.TEST_SESSION,
         1,
         ciProvider,
-        config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null);
+        config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null,
+        config.isAgentlessLogSubmissionEnabled() ? AgentlessLogSubmissionEnabled.TRUE : null,
+        CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(config.getCiVisibilityTestOrder())
+            ? FailFastTestOrderEnabled.TRUE
+            : null);
 
     if (instrumentationType == InstrumentationType.MANUAL_API) {
       metricCollector.add(CiVisibilityCountMetric.MANUAL_API_EVENTS, 1, EventType.SESSION);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

@@ -25,6 +25,8 @@ TestSuiteImpl testSuiteStart(
    */
   boolean isNew(TestIdentifier test);
 
+  boolean isFlaky(TestIdentifier test);
+
   /**
    * Checks if a given test should be skipped with Intelligent Test Runner or not
    *

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

@@ -87,6 +87,11 @@ public boolean isNew(TestIdentifier test) {
     return executionStrategy.isNew(test);
   }
 
+  @Override
+  public boolean isFlaky(TestIdentifier test) {
+    return executionStrategy.isFlaky(test);
+  }
+
   @Override
   public boolean shouldBeSkipped(TestIdentifier test) {
     return executionStrategy.shouldBeSkipped(test);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

@@ -72,6 +72,11 @@ public boolean isNew(TestIdentifier test) {
     return executionStrategy.isNew(test);
   }
 
+  @Override
+  public boolean isFlaky(TestIdentifier test) {
+    return executionStrategy.isFlaky(test);
+  }
+
   @Override
   public boolean shouldBeSkipped(TestIdentifier test) {
     return executionStrategy.shouldBeSkipped(test);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

@@ -272,6 +272,16 @@ public TestRetryPolicy retryPolicy(TestIdentifier test) {
     return testModule.retryPolicy(test);
   }
 
+  @Override
+  public boolean isNew(TestIdentifier test) {
+    return testModule.isNew(test);
+  }
+
+  @Override
+  public boolean isFlaky(TestIdentifier test) {
+    return testModule.isFlaky(test);
+  }
+
   @Override
   public void close() {
     testModule.end(null);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java

@@ -43,6 +43,11 @@ public boolean isNew(TestIdentifier test) {
     return knownTests != null && !knownTests.contains(test.withoutParameters());
   }
 
+  public boolean isFlaky(TestIdentifier test) {
+    Collection<TestIdentifier> flakyTests = executionSettings.getFlakyTests();
+    return flakyTests != null && flakyTests.contains(test.withoutParameters());
+  }
+
   public boolean shouldBeSkipped(TestIdentifier test) {
     if (test == null) {
       return false;

dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

@@ -5,6 +5,7 @@ import datadog.communication.serialization.GrowableBuffer
 import datadog.communication.serialization.msgpack.MsgPackWriter
 import datadog.trace.agent.test.AgentTestRunner
 import datadog.trace.api.Config
+import datadog.trace.api.civisibility.CIConstants
 import datadog.trace.api.civisibility.DDTest
 import datadog.trace.api.civisibility.DDTestSuite
 import datadog.trace.api.civisibility.InstrumentationBridge
@@ -124,7 +125,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
         skippableTestsWithMetadata,
         [:],
         flakyTests,
-        earlyFlakinessDetectionEnabled ? knownTests : null)
+        earlyFlakinessDetectionEnabled || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(Config.get().ciVisibilityTestOrder) ? knownTests : null)
       }
     }
 
@@ -224,12 +225,22 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
 
   def givenFlakyTests(List<TestIdentifier> tests) {
     flakyTests.addAll(tests)
-    flakyRetryEnabled = true
   }
 
   def givenKnownTests(List<TestIdentifier> tests) {
     knownTests.addAll(tests)
-    earlyFlakinessDetectionEnabled = true
+  }
+
+  def givenFlakyRetryEnabled(boolean flakyRetryEnabled) {
+    this.flakyRetryEnabled = flakyRetryEnabled
+  }
+
+  def givenEarlyFlakinessDetectionEnabled(boolean earlyFlakinessDetectionEnabled) {
+    this.earlyFlakinessDetectionEnabled = earlyFlakinessDetectionEnabled
+  }
+
+  def givenTestsOrder(String testsOrder) {
+    injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder)
   }
 
   @Override
@@ -274,6 +285,33 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
     return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
   }
 
+  def assertTestsOrder(List<TestIdentifier> expectedOrder) {
+    TEST_WRITER.waitForTraces(expectedOrder.size() + 1)
+    def traces = TEST_WRITER.toList()
+    def events = getEventsAsJson(traces)
+    def identifiers = getTestIdentifiers(events)
+    if (identifiers != expectedOrder) {
+      throw new AssertionError("Expected order: $expectedOrder, but got: $identifiers")
+    }
+    return true
+  }
+
+  def getTestIdentifiers(List<Map> events) {
+    events.sort(Comparator.comparing { it['content']['start'] as Long })
+    def testIdentifiers = []
+    for (Map event : events) {
+      if (event['content']['meta']['test.name']) {
+        testIdentifiers.add(test(event['content']['meta']['test.suite'] as String, event['content']['meta']['test.name'] as String))
+      }
+    }
+    return testIdentifiers
+  }
+
+  def test(String suite, String name, String parameters = null) {
+
+    return new TestIdentifier(suite, name, parameters)
+  }
+
   def getEventsAsJson(List<List<DDSpan>> traces) {
     return getSpansAsJson(new CiTestCycleMapperV1(Config.get().getCiVisibilityWellKnownTags(), false), traces)
   }

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy

@@ -53,6 +53,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
   }
 
   def "test flaky retries #testcaseName"() {
+    givenFlakyRetryEnabled(true)
     givenFlakyTests(retriedTests)
 
     runFeatures(features)
@@ -71,6 +72,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
   }
 
   def "test early flakiness detection #testcaseName"() {
+    givenEarlyFlakinessDetectionEnabled(true)
     givenKnownTests(knownTestsList)
 
     runFeatures(features)

dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy

@@ -31,6 +31,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
   }
 
   def "test flaky retries #testcaseName"() {
+    givenFlakyRetryEnabled(true)
     givenFlakyTests(retriedTests)
 
     runTests(tests)
@@ -45,6 +46,7 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
   }
 
   def "test early flakiness detection #testcaseName"() {
+    givenEarlyFlakinessDetectionEnabled(true)
     givenKnownTests(knownTestsList)
 
     runTests(tests)

dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy

@@ -90,6 +90,7 @@ class JUnit4Test extends CiVisibilityInstrumentationTest {
   }
 
   def "test flaky retries #testcaseName"() {
+    givenFlakyRetryEnabled(true)
     givenFlakyTests(retriedTests)
 
     runTests(tests)
@@ -110,6 +111,7 @@ class JUnit4Test extends CiVisibilityInstrumentationTest {
   }
 
   def "test early flakiness detection #testcaseName"() {
+    givenEarlyFlakinessDetectionEnabled(true)
     givenKnownTests(knownTestsList)
 
     runTests(tests)

dd-java-agent/instrumentation/junit-5.3/build.gradle

@@ -33,9 +33,9 @@ dependencies {
 
   // versions used below are not the minimum ones that we support,
   // but the tests need to use them in order to be compliant with Spock 2.x
-  testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2'
-  testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2'
-  testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2'
+  testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2'
+  testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2'
+  testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2'
 
   latestDepTestImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: '+'
   latestDepTestImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '+'
@@ -44,8 +44,8 @@ dependencies {
 
 configurations.matching({ it.name.startsWith('test') }).each({
   it.resolutionStrategy {
-    force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.8.2'
-    force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.2'
-    force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.2'
+    force group: 'org.junit.platform', name: 'junit-platform-launcher', version: '1.9.2'
+    force group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.9.2'
+    force group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.9.2'
   }
 })

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/build.gradle

@@ -13,6 +13,7 @@ addTestSuiteForDir('latestDepTest', 'test')
 
 dependencies {
   implementation project(':dd-java-agent:instrumentation:junit-5.3')
+
   compileOnly group: 'io.cucumber', name: 'cucumber-junit-platform-engine', version: '5.4.0'
   compileOnly group: 'io.cucumber', name: 'cucumber-java', version: '5.4.0'
 

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java

@@ -7,17 +7,12 @@
 import com.google.auto.service.AutoService;
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
-import datadog.trace.bootstrap.ContextStore;
-import datadog.trace.bootstrap.InstrumentationContext;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import io.cucumber.junit.platform.engine.CucumberTestEngine;
-import java.util.Collections;
-import java.util.Map;
 import net.bytebuddy.asm.Advice;
 import net.bytebuddy.matcher.ElementMatcher;
 import org.junit.platform.engine.EngineExecutionListener;
 import org.junit.platform.engine.ExecutionRequest;
-import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.TestEngine;
 import org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService;
 
@@ -51,11 +46,6 @@ public String[] helperClassNames() {
     };
   }
 
-  @Override
-  public Map<String, String> contextStore() {
-    return Collections.singletonMap("org.junit.platform.engine.TestDescriptor", "java.lang.Object");
-  }
-
   @Override
   public void methodAdvice(MethodTransformer transformer) {
     transformer.applyAdvice(
@@ -87,12 +77,6 @@ public static void addTracingListener(
         return;
       }
 
-      ContextStore<TestDescriptor, Object> contextStore =
-          InstrumentationContext.get(TestDescriptor.class, Object.class);
-      TestEventsHandlerHolder.setContextStores(
-          (ContextStore) contextStore, (ContextStore) contextStore);
-      TestEventsHandlerHolder.start();
-
       CucumberTracingListener tracingListener = new CucumberTracingListener(testEngine);
       EngineExecutionListener originalListener = executionRequest.getEngineExecutionListener();
       EngineExecutionListener compositeListener =