Log test failures and errors

Tests run during discovery in the coverage phase did not report the exceptions if they errored or failed. Pitest 1.15.2 now allows this information to be passed on for logging.
pitest · Oct 24, 2023 · 3a40a2d · 3a40a2d
1 parent 9d66b69
commit 3a40a2d
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 19 deletions.
diff --git a/pom.xml b/pom.xml
@@ -23,7 +23,7 @@
 		<junit.platform.version>1.9.2</junit.platform.version>
 		<junit.version>5.9.2</junit.version>
 		<mockito.version>2.7.6</mockito.version>
-		<pitest.version>1.9.0</pitest.version>
+		<pitest.version>1.15.2</pitest.version>
 		<cucumber.version>5.0.0</cucumber.version>
 		<spock.version>2.3-groovy-4.0</spock.version>
 		<groovy.version>4.0.11</groovy.version>
@@ -238,7 +238,7 @@
 			<plugin>
 				<groupId>org.jacoco</groupId>
 				<artifactId>jacoco-maven-plugin</artifactId>
-				<version>0.8.5</version>
+				<version>0.8.11</version>
 				<executions>
 					<execution>
 						<goals>

diff --git a/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java b/src/main/java/org/pitest/junit5/JUnit5TestUnitFinder.java
@@ -130,9 +130,11 @@ public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult
                 if (!identifiers.contains(testIdentifier)) {
                     identifiers.add(testIdentifier);
                 }
-                l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass), false);
+                l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass)
+                        , false, testExecutionResult.getThrowable().orElse(null));
             } else if (testIdentifier.isTest()) {
-                l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass), true);
+                l.executionFinished(new Description(testIdentifier.getUniqueId(), testClass)
+                        , true);
             }
         }
 

diff --git a/src/test/java/org/pitest/junit5/JUnit5TestUnitFinderTest.java b/src/test/java/org/pitest/junit5/JUnit5TestUnitFinderTest.java
@@ -19,6 +19,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 
 import org.junit.jupiter.api.Test;
+import org.opentest4j.AssertionFailedError;
 import org.pitest.junit5.cucumber.RunCucumberTest;
 import org.pitest.junit5.repository.AbstractTestClass;
 import org.pitest.junit5.repository.InterfaceTestClass;
@@ -67,9 +68,11 @@
 import org.pitest.testapi.TestGroupConfig;
 import org.pitest.testapi.TestUnit;
 import org.pitest.testapi.TestUnitExecutionListener;
+import org.spockframework.runtime.ConditionNotSatisfiedError;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 /**
@@ -149,12 +152,14 @@ void detectsFailingTests() {
     void detectsFailingSpockTests() {
         findsAndRunsNTests(1, TestSpecWithFailingFeature.class);
         nTestsFails(1, TestSpecWithFailingFeature.class);
+        errorIsRecorded(t -> t instanceof ConditionNotSatisfiedError, TestSpecWithFailingFeature.class);
     }
 
     @Test
     void detectsErroringTestsWhenPassingTestsPresent() {
         nTestsPass(2, TestClassWithMixedPassAndFail.class);
         nTestsFails(2, TestClassWithMixedPassAndFail.class);
+        errorIsRecorded(t -> t instanceof RuntimeException, TestClassWithMixedPassAndFail.class);
     }
 
     @Test
@@ -265,6 +270,7 @@ void excludesTestsByTag() {
         findsAndRunsNTests(3, new JUnit5TestUnitFinder(new TestGroupConfig().withExcludedGroups("excluded"), emptyList()), TestClassWithTags.class);
     }
 
+    @Test
     void excludesSpockTestsByTag() {
         findsAndRunsNTests(3, new JUnit5TestUnitFinder(new TestGroupConfig().withExcludedGroups("excluded"), emptyList()), TestSpecWithTags.class);
     }
@@ -322,6 +328,7 @@ void findsAndRunsTestsWithSetupSpec() {
     @Test
     void findsAndRunsTestsWithFailingAfterAll() {
         findsAndRunsNTests(2, TestClassWithFailingAfterAll.class);
+        errorIsRecorded(t -> t instanceof AssertionFailedError, TestClassWithFailingAfterAll.class);
     }
 
     @Test
@@ -368,6 +375,11 @@ private void nTestsFails(int n, Class<?> clazz) {
         assertThat(l.failed).hasSize(n);
     }
 
+    private void errorIsRecorded(Predicate<Throwable> p, Class<?> clazz) {
+        RecordingListener l = run(basicConfig(), clazz);
+        assertThat(l.errors).anyMatch(p);
+    }
+
     private RecordingListener run(JUnit5TestUnitFinder underTest, Class<?> clazz) {
         RecordingListener l = new RecordingListener();
         underTest.findTestUnits(clazz, l);
@@ -384,33 +396,24 @@ class RecordingListener implements TestUnitExecutionListener {
     List<Description> started = new ArrayList<>();
     List<Description> failed = new ArrayList<>();
     List<Description> passed = new ArrayList<>();
-    TestUnitExecutionListener l = new TestUnitExecutionListener() {
-        @Override
-        public void executionStarted(Description description) {
-            started.add(description);
-        }
 
-        @Override
-        public void executionFinished(Description description, boolean pass) {
-            if (pass) {
-                passed.add(description);
-            } else {
-                failed.add(description);
-            }
-        }
-    };
+    List<Throwable> errors = new ArrayList<>();
 
     @Override
     public void executionStarted(Description description) {
         started.add(description);
     }
 
     @Override
-    public void executionFinished(Description description, boolean pass) {
+    public void executionFinished(Description description, boolean pass, Throwable optional) {
         if (pass) {
             passed.add(description);
         } else {
             failed.add(description);
         }
+
+        if (optional != null) {
+            errors.add(optional);
+        }
     }
 }