From 7233e4c1fdb533e1b4479d660f591829d02de0b7 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 11 Nov 2024 14:09:19 +0300 Subject: [PATCH 1/8] Client version upgrade --- CHANGELOG.md | 2 ++ build.gradle | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c85ecc..c0f1ab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Changed +- Client version upgraded on [5.2.20](https://github.com/reportportal/client-java/releases/tag/5.2.20), by @HardNorth ## [5.4.3] ### Changed diff --git a/build.gradle b/build.gradle index b70a545..08fb5a1 100644 --- a/build.gradle +++ b/build.gradle @@ -39,7 +39,7 @@ repositories { } dependencies { - api 'com.epam.reportportal:client-java:5.2.15' + api 'com.epam.reportportal:client-java:5.2.20' compileOnly "org.testng:testng:${testng_version}" implementation 'org.slf4j:slf4j-api:2.0.4' From f3261981abba6b9f5c776c84b0c6b36aaa29c32b Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 11 Nov 2024 14:11:57 +0300 Subject: [PATCH 2/8] Avoid deprecated code --- src/main/java/com/epam/reportportal/testng/TestNGService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/epam/reportportal/testng/TestNGService.java b/src/main/java/com/epam/reportportal/testng/TestNGService.java index 77431c3..3870cad 100644 --- a/src/main/java/com/epam/reportportal/testng/TestNGService.java +++ b/src/main/java/com/epam/reportportal/testng/TestNGService.java @@ -31,7 +31,7 @@ import com.epam.reportportal.utils.MemoizingSupplier; import com.epam.reportportal.utils.ParameterUtils; import com.epam.reportportal.utils.TestCaseIdUtils; -import com.epam.reportportal.utils.markdown.MarkdownUtils; +import com.epam.reportportal.utils.formatting.MarkdownUtils; import com.epam.reportportal.utils.properties.SystemAttributesExtractor; import com.epam.ta.reportportal.ws.model.*; import com.epam.ta.reportportal.ws.model.attribute.ItemAttributesRQ; From 9fd7acfaf4ba21d889013e9ff17e9d3ea4bfe97d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 11 Nov 2024 14:35:13 +0300 Subject: [PATCH 3/8] Common Stack Trace frames skip in description and logs --- CHANGELOG.md | 2 + .../reportportal/testng/TestNGService.java | 66 ++++++++++--------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0f1ab8..9832bdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Added +- Common Stack Trace frames skip in description and logs, by @HardNorth ### Changed - Client version upgraded on [5.2.20](https://github.com/reportportal/client-java/releases/tag/5.2.20), by @HardNorth diff --git a/src/main/java/com/epam/reportportal/testng/TestNGService.java b/src/main/java/com/epam/reportportal/testng/TestNGService.java index 3870cad..3ae6cfe 100644 --- a/src/main/java/com/epam/reportportal/testng/TestNGService.java +++ b/src/main/java/com/epam/reportportal/testng/TestNGService.java @@ -39,7 +39,6 @@ import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; import io.reactivex.Maybe; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.tuple.Pair; import org.testng.*; import org.testng.annotations.Factory; @@ -65,11 +64,11 @@ import java.util.stream.Stream; import static com.epam.reportportal.testng.util.ItemTreeUtils.createKey; +import static com.epam.reportportal.utils.formatting.ExceptionUtils.getStackTrace; import static java.util.Optional.ofNullable; import static java.util.stream.Collectors.toList; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace; /** * TestNG service implements operations for interaction ReportPortal @@ -299,6 +298,21 @@ public void startConfiguration(ITestResult testResult) { testResult.setAttribute(RP_ID, itemID); } + /** + * Extension point to customize test step description + * + * @param testResult TestNG's testResult context + * @return Test/Step Description being sent to ReportPortal + */ + @Nonnull + protected String createStepDescription(@Nonnull ITestResult testResult) { + var methodDescriptionOptional = getMethodAnnotation(Description.class, testResult); + if (methodDescriptionOptional.isPresent()) { + return methodDescriptionOptional.get().value(); + } + return testResult.getMethod().getDescription(); + } + /** * Extension point to customize test step creation event/request * @@ -364,6 +378,25 @@ public void startTestMethod(@Nonnull ITestResult testResult) { } } + /** + * Extension point to customize test step description with error message + * + * @param testResult TestNG's testResult context + * @return Test/Step Description being sent to ReportPortal + */ + @Nullable + private String getLogMessage(@Nonnull ITestResult testResult) { + String error = ofNullable(testResult.getThrowable()).map(t -> String.format( + DESCRIPTION_ERROR_FORMAT, + getStackTrace(t, new Throwable()) + )).orElse(null); + if (error == null) { + return null; + } + String description = createStepDescription(testResult); + return StringUtils.isNotBlank(description) ? MarkdownUtils.asTwoParts(description, error) : error; + } + /** * Extension point to customize test method on it's finish * @@ -494,7 +527,7 @@ public void sendReportPortalMsg(final ITestResult result) { rq.setItemUuid(itemUuid); rq.setLevel("ERROR"); if (result.getThrowable() != null) { - rq.setMessage(getStackTrace(result.getThrowable())); + rq.setMessage(getStackTrace(result.getThrowable(), new Throwable())); } else { rq.setMessage("Test has failed without exception"); } @@ -718,20 +751,6 @@ protected String createStepName(ITestResult testResult) { return testStepName == null ? testResult.getMethod().getMethodName() : testStepName; } - /** - * Extension point to customize test step description - * - * @param testResult TestNG's testResult context - * @return Test/Step Description being sent to ReportPortal - */ - protected String createStepDescription(ITestResult testResult) { - var methodDescriptionOptional = getMethodAnnotation(Description.class, testResult); - if (methodDescriptionOptional.isPresent()) { - return methodDescriptionOptional.get().value(); - } - return testResult.getMethod().getDescription(); - } - @Nullable private TestCaseIdEntry getTestCaseId(@Nonnull String codeRef, @Nonnull ITestResult testResult) { Method method = getMethod(testResult); @@ -801,17 +820,4 @@ Maybe getConfigParent(ITestResult testResult, TestMethodType type) { } return parentId; } - - /** - * Extension point to customize test step description with error message - * - * @param testResult TestNG's testResult context - * @return Test/Step Description being sent to ReportPortal - */ - private String getLogMessage(ITestResult testResult) { - String error = String.format(DESCRIPTION_ERROR_FORMAT, ExceptionUtils.getStackTrace(testResult.getThrowable())).trim(); - return ofNullable(createStepDescription(testResult)).filter(StringUtils::isNotBlank) - .map(description -> MarkdownUtils.asTwoParts(description, error)) - .orElse(error); - } } From f15580cfcde344534cabc26396d87f01e71498fd Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 11 Nov 2024 14:38:12 +0300 Subject: [PATCH 4/8] Fix build --- .../epam/reportportal/testng/TestDescriptionFailedTest.java | 2 +- .../reportportal/testng/TestLaunchFinishShutdownHook.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/epam/reportportal/testng/TestDescriptionFailedTest.java b/src/test/java/com/epam/reportportal/testng/TestDescriptionFailedTest.java index 48a457c..798fc6c 100644 --- a/src/test/java/com/epam/reportportal/testng/TestDescriptionFailedTest.java +++ b/src/test/java/com/epam/reportportal/testng/TestDescriptionFailedTest.java @@ -24,7 +24,7 @@ import com.epam.reportportal.service.ReportPortalClient; import com.epam.reportportal.testng.integration.feature.description.DescriptionFailedTest; import com.epam.reportportal.utils.MemoizingSupplier; -import com.epam.reportportal.utils.markdown.MarkdownUtils; +import com.epam.reportportal.utils.formatting.MarkdownUtils; import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; import com.epam.ta.reportportal.ws.model.launch.StartLaunchRQ; import java.util.Calendar; diff --git a/src/test/java/com/epam/reportportal/testng/TestLaunchFinishShutdownHook.java b/src/test/java/com/epam/reportportal/testng/TestLaunchFinishShutdownHook.java index 41792e7..8aa7165 100644 --- a/src/test/java/com/epam/reportportal/testng/TestLaunchFinishShutdownHook.java +++ b/src/test/java/com/epam/reportportal/testng/TestLaunchFinishShutdownHook.java @@ -1,6 +1,6 @@ package com.epam.reportportal.testng; -import com.epam.reportportal.service.statistics.StatisticsService; +import com.epam.reportportal.service.LaunchImpl; import com.epam.reportportal.testng.integration.feature.shutdown.LaunchFinishShutdownHookRemoveTest; import com.epam.reportportal.testng.integration.feature.shutdown.LaunchFinishShutdownHookTest; import com.epam.reportportal.util.test.ProcessUtils; @@ -29,7 +29,7 @@ public void test_shutdown_hook_finishes_launch_on_java_machine_exit(final Class< ServerSocket ss = SocketUtils.getServerSocketOnFreePort(); SocketUtils.ServerCallable serverCallable = new SocketUtils.ServerCallable(ss, Collections.emptyMap(), "files/socket_response.txt"); Callable clientCallable = () -> ProcessUtils.buildProcess(true, clazz, - Collections.singletonMap(StatisticsService.DISABLE_PROPERTY, "1"), String.valueOf(ss.getLocalPort())); + Collections.singletonMap(LaunchImpl.DISABLE_PROPERTY, "1"), String.valueOf(ss.getLocalPort())); Pair, Process> startResult = SocketUtils.executeServerCallable(serverCallable, clientCallable, 15); assertThat(startResult.getValue(), notNullValue()); assertThat("First request is a launch start", startResult.getKey().get(0), startsWith("POST /api/v2/test-project/launch")); From edbd871229d712f958f6e83e10ed707f610e3a11 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 12 Nov 2024 13:37:25 +0300 Subject: [PATCH 5/8] `@Issue` and `@Issues` annotations support --- CHANGELOG.md | 1 + .../reportportal/testng/TestNGService.java | 35 ++++++++++--------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9832bdb..60e0bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Added - Common Stack Trace frames skip in description and logs, by @HardNorth +- `@Issue` and `@Issues` annotations support, by @HardNorth ### Changed - Client version upgraded on [5.2.20](https://github.com/reportportal/client-java/releases/tag/5.2.20), by @HardNorth diff --git a/src/main/java/com/epam/reportportal/testng/TestNGService.java b/src/main/java/com/epam/reportportal/testng/TestNGService.java index 3ae6cfe..8027ce9 100644 --- a/src/main/java/com/epam/reportportal/testng/TestNGService.java +++ b/src/main/java/com/epam/reportportal/testng/TestNGService.java @@ -15,10 +15,7 @@ */ package com.epam.reportportal.testng; -import com.epam.reportportal.annotations.Description; -import com.epam.reportportal.annotations.DisplayName; -import com.epam.reportportal.annotations.ParameterKey; -import com.epam.reportportal.annotations.TestCaseId; +import com.epam.reportportal.annotations.*; import com.epam.reportportal.annotations.attribute.Attributes; import com.epam.reportportal.listeners.ItemStatus; import com.epam.reportportal.listeners.ListenerParameters; @@ -27,10 +24,7 @@ import com.epam.reportportal.service.item.TestCaseIdEntry; import com.epam.reportportal.service.tree.TestItemTree; import com.epam.reportportal.testng.util.internal.LimitedSizeConcurrentHashMap; -import com.epam.reportportal.utils.AttributeParser; -import com.epam.reportportal.utils.MemoizingSupplier; -import com.epam.reportportal.utils.ParameterUtils; -import com.epam.reportportal.utils.TestCaseIdUtils; +import com.epam.reportportal.utils.*; import com.epam.reportportal.utils.formatting.MarkdownUtils; import com.epam.reportportal.utils.properties.SystemAttributesExtractor; import com.epam.ta.reportportal.ws.model.*; @@ -386,8 +380,7 @@ public void startTestMethod(@Nonnull ITestResult testResult) { */ @Nullable private String getLogMessage(@Nonnull ITestResult testResult) { - String error = ofNullable(testResult.getThrowable()).map(t -> String.format( - DESCRIPTION_ERROR_FORMAT, + String error = ofNullable(testResult.getThrowable()).map(t -> String.format(DESCRIPTION_ERROR_FORMAT, getStackTrace(t, new Throwable()) )).orElse(null); if (error == null) { @@ -479,6 +472,15 @@ private void processFinishRetryFlag(ITestResult testResult, FinishTestItemRQ rq) protected void createSkippedSteps(ITestResult testResult) { } + @Nullable + protected com.epam.ta.reportportal.ws.model.issue.Issue createIssue(@Nonnull ITestResult testResult) { + String stepName = createStepName(testResult); + List parameters = createStepParameters(testResult); + return getMethodAnnotation(Issues.class, testResult).map(i -> IssueUtils.createIssue(i, stepName, parameters)) + .orElseGet(() -> getMethodAnnotation(Issue.class, testResult).map(i -> IssueUtils.createIssue(i, stepName, parameters)) + .orElse(null)); + } + @Override public void finishTestMethod(ItemStatus status, ITestResult testResult) { Maybe itemId = getAttribute(testResult, RP_ID); @@ -514,6 +516,10 @@ && getAttribute(testResult, RP_RETRY) != null))) { processFinishRetryFlag(testResult, rq); + if (type == TestMethodType.STEP) { + rq.setIssue(createIssue(testResult)); + } + Maybe finishItemResponse = launch.get().finishTestItem(itemId, rq); if (launch.get().getParameters().isCallbackReportingEnabled()) { updateTestItemTree(finishItemResponse, testResult); @@ -758,13 +764,8 @@ private TestCaseIdEntry getTestCaseId(@Nonnull String codeRef, @Nonnull ITestRes List parameters = ofNullable(testResult.getParameters()).map(Arrays::asList).orElse(null); TestCaseIdEntry id = getMethodAnnotation(TestCaseId.class, testResult - ).flatMap(a -> ofNullable(method).map(m -> TestCaseIdUtils.getTestCaseId( - a, - m, - codeRef, - parameters, - instance - ))).orElse(TestCaseIdUtils.getTestCaseId(codeRef, parameters)); + ).flatMap(a -> ofNullable(method).map(m -> TestCaseIdUtils.getTestCaseId(a, m, codeRef, parameters, instance))) + .orElse(TestCaseIdUtils.getTestCaseId(codeRef, parameters)); return id == null ? null : id.getId().endsWith("[]") ? new TestCaseIdEntry(id.getId().substring(0, id.getId().length() - 2)) : id; } From 6553e8fe32e0ca7a955f2aeb7e5ee0b54e83774c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 12 Nov 2024 14:02:29 +0300 Subject: [PATCH 6/8] Fix tests --- .../epam/reportportal/testng/TestNGService.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/epam/reportportal/testng/TestNGService.java b/src/main/java/com/epam/reportportal/testng/TestNGService.java index 8027ce9..e353b74 100644 --- a/src/main/java/com/epam/reportportal/testng/TestNGService.java +++ b/src/main/java/com/epam/reportportal/testng/TestNGService.java @@ -435,7 +435,7 @@ private void processFinishRetryFlag(ITestResult testResult, FinishTestItemRQ rq) TestMethodType type = getAttribute(testResult, RP_METHOD_TYPE); boolean isRetried = testResult.wasRetried(); - if (TestMethodType.STEP == type && getAttribute(testResult, RP_RETRY) == null && isRetried) { + if (TestMethodType.STEP == type && getAttribute(testResult, RP_RETRY) == null && isRetried && rq.getIssue() == null) { RETRY_STATUS_TRACKER.put(instance, Boolean.TRUE); rq.setRetry(Boolean.TRUE); rq.setIssue(Launch.NOT_ISSUE); @@ -498,14 +498,18 @@ public void finishTestMethod(ItemStatus status, ITestResult testResult) { TestMethodType type = getAttribute(testResult, RP_METHOD_TYPE); Object instance = testResult.getInstance(); + if (type == TestMethodType.STEP) { + rq.setIssue(createIssue(testResult)); + } + // TestNG does not repeat before methods if an after method fails during retries. But reports them as skipped. // Mark before methods as not an issue if it is not a culprit. if (instance != null) { if (ItemStatus.FAILED == status && (TestMethodType.BEFORE_METHOD == type || TestMethodType.BEFORE_CLASS == type)) { SKIPPED_STATUS_TRACKER.put(instance, Boolean.TRUE); } - if (ItemStatus.SKIPPED == status && (SKIPPED_STATUS_TRACKER.containsKey(instance) || (TestMethodType.BEFORE_METHOD == type - && getAttribute(testResult, RP_RETRY) != null))) { + if (status == ItemStatus.SKIPPED && rq.getIssue() == null + && (SKIPPED_STATUS_TRACKER.containsKey(instance) || (TestMethodType.BEFORE_METHOD == type && getAttribute(testResult, RP_RETRY) != null))) { rq.setIssue(Launch.NOT_ISSUE); } if (ItemStatus.SKIPPED == status && BEFORE_METHODS.contains(type) && testResult.getThrowable() != null) { @@ -516,10 +520,6 @@ && getAttribute(testResult, RP_RETRY) != null))) { processFinishRetryFlag(testResult, rq); - if (type == TestMethodType.STEP) { - rq.setIssue(createIssue(testResult)); - } - Maybe finishItemResponse = launch.get().finishTestItem(itemId, rq); if (launch.get().getParameters().isCallbackReportingEnabled()) { updateTestItemTree(finishItemResponse, testResult); From e5390daf2ba6bcc2ec8dd25e818e4f07f91f5a52 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 13 Nov 2024 13:41:56 +0300 Subject: [PATCH 7/8] Add test examples --- .../issue/ParameterizedWithOneIssueTest.java | 42 ++++++++++++++++++ .../issue/ParameterizedWithTwoIssueTest.java | 43 +++++++++++++++++++ .../feature/issue/SimpleIssueTest.java | 31 +++++++++++++ .../feature/issue/SimpleSkippedIssueTest.java | 31 +++++++++++++ .../feature/issue/SimpleTwoIssuesTest.java | 32 ++++++++++++++ 5 files changed, 179 insertions(+) create mode 100644 src/test/java/com/epam/reportportal/testng/integration/feature/issue/ParameterizedWithOneIssueTest.java create mode 100644 src/test/java/com/epam/reportportal/testng/integration/feature/issue/ParameterizedWithTwoIssueTest.java create mode 100644 src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleIssueTest.java create mode 100644 src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleSkippedIssueTest.java create mode 100644 src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleTwoIssuesTest.java diff --git a/src/test/java/com/epam/reportportal/testng/integration/feature/issue/ParameterizedWithOneIssueTest.java b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/ParameterizedWithOneIssueTest.java new file mode 100644 index 0000000..50f1c90 --- /dev/null +++ b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/ParameterizedWithOneIssueTest.java @@ -0,0 +1,42 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.testng.integration.feature.issue; + +import com.epam.reportportal.annotations.Issue; +import com.epam.reportportal.annotations.TestFilter; +import com.epam.reportportal.annotations.TestParamFilter; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class ParameterizedWithOneIssueTest { + + public static final String FAILURE_MESSAGE = "This parameterized test is expected to fail: "; + public static final String ISSUE_MESSAGE = "This test is expected to fail"; + + public static final Object[][] PARAMS = { { true }, { false } }; + + @DataProvider + public static Object[][] paramsProvider() { + return PARAMS; + } + + @Test(dataProvider = "paramsProvider") + @Issue(value = "ab001", comment = ISSUE_MESSAGE, filter = @TestFilter(param = { @TestParamFilter(valueStartsWith = "false") })) + public void failureTest(boolean param) { + throw new IllegalStateException(FAILURE_MESSAGE + param); + } +} diff --git a/src/test/java/com/epam/reportportal/testng/integration/feature/issue/ParameterizedWithTwoIssueTest.java b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/ParameterizedWithTwoIssueTest.java new file mode 100644 index 0000000..6ddaf0e --- /dev/null +++ b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/ParameterizedWithTwoIssueTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.testng.integration.feature.issue; + +import com.epam.reportportal.annotations.Issue; +import com.epam.reportportal.annotations.TestFilter; +import com.epam.reportportal.annotations.TestParamFilter; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class ParameterizedWithTwoIssueTest { + + public static final String FAILURE_MESSAGE = "This parameterized test is expected to fail: "; + public static final String ISSUE_MESSAGE = "This test is expected to fail"; + + public static final Object[][] PARAMS = { { true }, { false } }; + + @DataProvider + public static Object[][] paramsProvider() { + return PARAMS; + } + + @Test(dataProvider = "paramsProvider") + @Issue(value = "ab001", comment = ISSUE_MESSAGE, filter = @TestFilter(param = { @TestParamFilter(valueStartsWith = "true") })) + @Issue(value = "pb001", comment = ISSUE_MESSAGE, filter = @TestFilter(param = { @TestParamFilter(valueStartsWith = "false") })) + public void failureTest(boolean param) { + throw new IllegalStateException(FAILURE_MESSAGE + param); + } +} diff --git a/src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleIssueTest.java b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleIssueTest.java new file mode 100644 index 0000000..c72791c --- /dev/null +++ b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleIssueTest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.testng.integration.feature.issue; + +import com.epam.reportportal.annotations.Issue; +import org.testng.annotations.Test; + +public class SimpleIssueTest { + + public static final String FAILURE_MESSAGE = "This test is expected to fail"; + + @Test + @Issue(value = "pb001", comment = FAILURE_MESSAGE) + public void failureTest() { + throw new IllegalStateException(FAILURE_MESSAGE); + } +} diff --git a/src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleSkippedIssueTest.java b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleSkippedIssueTest.java new file mode 100644 index 0000000..2656c7e --- /dev/null +++ b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleSkippedIssueTest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.testng.integration.feature.issue; + +import com.epam.reportportal.annotations.Issue; +import org.testng.annotations.Test; + +public class SimpleSkippedIssueTest { + + public static final String FAILURE_MESSAGE = "This test is expected to fail"; + + @Test(enabled = false) + @Issue(value = "pb001", comment = FAILURE_MESSAGE) + public void failureTest() { + throw new IllegalStateException(FAILURE_MESSAGE); + } +} diff --git a/src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleTwoIssuesTest.java b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleTwoIssuesTest.java new file mode 100644 index 0000000..c572aef --- /dev/null +++ b/src/test/java/com/epam/reportportal/testng/integration/feature/issue/SimpleTwoIssuesTest.java @@ -0,0 +1,32 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.testng.integration.feature.issue; + +import com.epam.reportportal.annotations.Issue; +import org.testng.annotations.Test; + +public class SimpleTwoIssuesTest { + + public static final String FAILURE_MESSAGE = "This test is expected to fail"; + + @Test + @Issue(value = "ab001", comment = FAILURE_MESSAGE) + @Issue(value = "pb001", comment = FAILURE_MESSAGE) + public void failureTest() { + throw new IllegalStateException(FAILURE_MESSAGE); + } +} From 8e40365c918a344c1ff763bbbb4e47976d128c2f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 13 Nov 2024 14:26:21 +0300 Subject: [PATCH 8/8] Add `@Issue` annotation tests --- .../testng/IssueReportingTest.java | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 src/test/java/com/epam/reportportal/testng/IssueReportingTest.java diff --git a/src/test/java/com/epam/reportportal/testng/IssueReportingTest.java b/src/test/java/com/epam/reportportal/testng/IssueReportingTest.java new file mode 100644 index 0000000..17fac0a --- /dev/null +++ b/src/test/java/com/epam/reportportal/testng/IssueReportingTest.java @@ -0,0 +1,170 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.testng; + +import com.epam.reportportal.listeners.ItemStatus; +import com.epam.reportportal.listeners.ListenerParameters; +import com.epam.reportportal.service.Launch; +import com.epam.reportportal.testng.integration.feature.issue.ParameterizedWithOneIssueTest; +import com.epam.reportportal.testng.integration.feature.issue.ParameterizedWithTwoIssueTest; +import com.epam.reportportal.testng.integration.feature.issue.SimpleIssueTest; +import com.epam.reportportal.testng.integration.feature.issue.SimpleTwoIssuesTest; +import com.epam.reportportal.testng.integration.util.TestUtils; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.epam.ta.reportportal.ws.model.OperationCompletionRS; +import com.epam.ta.reportportal.ws.model.issue.Issue; +import io.reactivex.Maybe; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.stubbing.Answer; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.Queue; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class IssueReportingTest { + public static class TestReportPortalListener extends BaseTestNGListener { + public static final ThreadLocal LAUNCH_THREAD_LOCAL = new ThreadLocal<>(); + + public TestReportPortalListener() { + super(new TestNGService(LAUNCH_THREAD_LOCAL::get)); + } + + public static void initLaunch(Launch launch) { + LAUNCH_THREAD_LOCAL.set(launch); + } + + public static Launch getLaunch() { + return LAUNCH_THREAD_LOCAL.get(); + } + } + + private final String suiteId = CommonUtils.namedId("suite_"); + private final Maybe suiteMaybe = Maybe.just(suiteId); + private final String stepOneId = CommonUtils.namedId("step_"); + private final Maybe stepOneMaybe = Maybe.just(stepOneId); + private final String stepTwoId = CommonUtils.namedId("step_"); + private final Maybe stepTwoMaybe = Maybe.just(stepTwoId); + private final String stepThreeId = CommonUtils.namedId("step_"); + private final Maybe stepThreeMaybe = Maybe.just(stepThreeId); + private final Queue> stepIds = new LinkedList<>(Arrays.asList(stepOneMaybe, stepTwoMaybe, stepThreeMaybe)); + + @Mock + private Launch launch; + @Mock + private ListenerParameters parameters; + + @BeforeEach + public void initMocks() { + when(launch.getParameters()).thenReturn(parameters); + when(launch.startTestItem(any())).thenReturn(suiteMaybe); + when(launch.startTestItem(any(), any())).thenAnswer((Answer>) invocation -> CommonUtils.createMaybeUuid()); + when(launch.startTestItem(same(suiteMaybe), any())).thenAnswer((Answer>) invocation -> stepIds.poll()); + when(launch.startTestItem(same(stepOneMaybe), any())).thenAnswer((Answer>) invocation -> stepIds.poll()); + when(launch.finishTestItem(any(), + any() + )).thenAnswer((Answer>) invocation -> Maybe.just(new OperationCompletionRS("OK"))); + TestReportPortalListener.initLaunch(launch); + when(parameters.isCallbackReportingEnabled()).thenReturn(Boolean.TRUE); + } + + @Test + public void verify_simple_test_failure() { + TestUtils.runTests(Collections.singletonList(TestReportPortalListener.class), SimpleIssueTest.class); + + ArgumentCaptor testCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(launch).finishTestItem(same(stepTwoMaybe), testCaptor.capture()); + + FinishTestItemRQ finishTestItemRQ = testCaptor.getValue(); + assertThat(finishTestItemRQ.getStatus(), equalTo(ItemStatus.FAILED.name())); + assertThat(finishTestItemRQ.getIssue(), notNullValue()); + Issue issue = finishTestItemRQ.getIssue(); + assertThat(issue.getIssueType(), equalTo("pb001")); + assertThat(issue.getComment(), equalTo(SimpleIssueTest.FAILURE_MESSAGE)); + } + + @Test + public void verify_test_failure_with_two_issues() { + TestUtils.runTests(Collections.singletonList(TestReportPortalListener.class), SimpleTwoIssuesTest.class); + + ArgumentCaptor testCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(launch).finishTestItem(same(stepTwoMaybe), testCaptor.capture()); + + FinishTestItemRQ finishTestItemRQ = testCaptor.getValue(); + assertThat(finishTestItemRQ.getStatus(), equalTo(ItemStatus.FAILED.name())); + assertThat(finishTestItemRQ.getIssue(), notNullValue()); + Issue issue = finishTestItemRQ.getIssue(); + assertThat(issue.getIssueType(), equalTo("ab001")); + assertThat(issue.getComment(), equalTo(SimpleTwoIssuesTest.FAILURE_MESSAGE)); + } + + @Test + public void verify_parameterized_test_failure_with_one_issue() { + TestUtils.runTests(Collections.singletonList(TestReportPortalListener.class), ParameterizedWithOneIssueTest.class); + + ArgumentCaptor firstTestCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(launch).finishTestItem(same(stepTwoMaybe), firstTestCaptor.capture()); + ArgumentCaptor secondTestCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(launch).finishTestItem(same(stepThreeMaybe), secondTestCaptor.capture()); + + FinishTestItemRQ finishTestItemRQ = firstTestCaptor.getValue(); + assertThat(finishTestItemRQ.getStatus(), equalTo(ItemStatus.FAILED.name())); + assertThat(finishTestItemRQ.getIssue(), nullValue()); + + finishTestItemRQ = secondTestCaptor.getValue(); + assertThat(finishTestItemRQ.getStatus(), equalTo(ItemStatus.FAILED.name())); + assertThat(finishTestItemRQ.getIssue(), notNullValue()); + Issue issue = finishTestItemRQ.getIssue(); + assertThat(issue.getIssueType(), equalTo("ab001")); + assertThat(issue.getComment(), equalTo(ParameterizedWithOneIssueTest.ISSUE_MESSAGE)); + } + + @Test + public void verify_parameterized_test_failure_with_two_issues() { + TestUtils.runTests(Collections.singletonList(TestReportPortalListener.class), ParameterizedWithTwoIssueTest.class); + + ArgumentCaptor firstTestCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(launch).finishTestItem(same(stepTwoMaybe), firstTestCaptor.capture()); + ArgumentCaptor secondTestCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(launch).finishTestItem(same(stepThreeMaybe), secondTestCaptor.capture()); + + FinishTestItemRQ finishTestItemRQ = firstTestCaptor.getValue(); + assertThat(finishTestItemRQ.getStatus(), equalTo(ItemStatus.FAILED.name())); + assertThat(finishTestItemRQ.getIssue(), notNullValue()); + Issue issue = finishTestItemRQ.getIssue(); + assertThat(issue.getIssueType(), equalTo("ab001")); + assertThat(issue.getComment(), equalTo(ParameterizedWithTwoIssueTest.ISSUE_MESSAGE)); + + finishTestItemRQ = secondTestCaptor.getValue(); + assertThat(finishTestItemRQ.getStatus(), equalTo(ItemStatus.FAILED.name())); + assertThat(finishTestItemRQ.getIssue(), notNullValue()); + issue = finishTestItemRQ.getIssue(); + assertThat(issue.getIssueType(), equalTo("pb001")); + assertThat(issue.getComment(), equalTo(ParameterizedWithTwoIssueTest.ISSUE_MESSAGE)); + } +}