MsTest DataRow support

CHANGELOG.md

@@ -1,6 +1,7 @@
 # [vNext]
 
 * Support for PriorityAttribute in MsTest adapter
+* Support for Scenario Outline / DataRowAttribute in MsTest adapter
 
 # v1.0.1 - 2024-02-16
 

Reqnroll.Generator/UnitTestProvider/MsTestV2GeneratorProvider.cs

@@ -16,6 +16,7 @@ public class MsTestV2GeneratorProvider : MsTestGeneratorProvider
         protected internal const string PRIORITY_ATTR = "Microsoft.VisualStudio.TestTools.UnitTesting.PriorityAttribute";
         protected internal const string WORKITEM_ATTR = "Microsoft.VisualStudio.TestTools.UnitTesting.WorkItemAttribute";
         protected internal const string DEPLOYMENTITEM_ATTR = "Microsoft.VisualStudio.TestTools.UnitTesting.DeploymentItemAttribute";
+        protected internal const string ROW_ATTR = "Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute";
         protected internal const string OWNER_TAG = "owner:";
         protected internal const string PRIORITY_TAG = "priority:";
         protected internal const string WORKITEM_TAG = "workitem:";
@@ -27,7 +28,42 @@ public MsTestV2GeneratorProvider(CodeDomHelper codeDomHelper) : base(codeDomHelp
 
         public override UnitTestGeneratorTraits GetTraits()
         {
-            return UnitTestGeneratorTraits.ParallelExecution;
+            return UnitTestGeneratorTraits.RowTests | UnitTestGeneratorTraits.ParallelExecution;
+        }
+
+        public override void SetRowTest(TestClassGenerationContext generationContext, CodeMemberMethod testMethod, string scenarioTitle)
+        {
+            SetTestMethod(generationContext, testMethod, scenarioTitle);
+        }
+
+        public override void SetRow(TestClassGenerationContext generationContext, CodeMemberMethod testMethod, IEnumerable<string> arguments, IEnumerable<string> tags, bool isIgnored)
+        {
+            // MsTest doesn't support to ignore a specific test case / DataRow
+            if (isIgnored)
+            {
+                return;
+            }
+
+            // MsTest doesn't support categories for a specific test case / DataRow
+            var args = arguments.Select(arg => new CodeAttributeArgument(new CodePrimitiveExpression(arg))).ToList();
+
+            var tagExpressions = tags.Select(t => (CodeExpression)new CodePrimitiveExpression(t)).ToArray();
+
+            // MsTest v2.* cannot handle string[] as the second argument of the [DataRow] attribute.
+            // To compensate this, we include a dummy parameter and argument in this case.
+            if (tagExpressions.Any() && args.Count <= 1)
+            {
+                args.Add(new CodeAttributeArgument(new CodePrimitiveExpression("")));
+                var dummyParameterName = "notUsed6248";
+                if (testMethod.Parameters.OfType<CodeParameterDeclarationExpression>().All(p => p.Name != dummyParameterName))
+                    testMethod.Parameters.Insert(testMethod.Parameters.Count - 1, new CodeParameterDeclarationExpression(typeof(string), dummyParameterName));
+            }
+
+            args.Add(new CodeAttributeArgument(tagExpressions.Any()
+               ? new CodeArrayCreateExpression(typeof(string[]), tagExpressions)
+               : new CodePrimitiveExpression(null)));
+
+            CodeDomHelper.AddAttribute(testMethod, ROW_ATTR, args.ToArray());
         }
 
         public override void SetTestClass(TestClassGenerationContext generationContext, string featureTitle, string featureDescription)

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TRXParser.cs

@@ -15,6 +15,7 @@ public class TRXParser
         private readonly XName _unitTestResultElementName;
         private readonly XName _unitTestResultOutputElementName;
         private readonly XName _unitTestResultStdOutElementName;
+        private readonly XName _unitTestResultInnerResultsElementName;
         private readonly XName _testRunElementName;
         private readonly XName _resultsElementName;
 
@@ -24,6 +25,7 @@ public TRXParser(TestRunConfiguration testRunConfiguration)
             _unitTestResultElementName = _xmlns + "UnitTestResult";
             _unitTestResultOutputElementName = _xmlns + "Output";
             _unitTestResultStdOutElementName = _xmlns + "StdOut";
+            _unitTestResultInnerResultsElementName = _xmlns + "InnerResults";
             _testRunElementName = _xmlns + "TestRun";
             _resultsElementName = _xmlns + "Results";
         }
@@ -61,12 +63,18 @@ private TestExecutionResult GetCommonTestExecutionResult(XDocument trx, string o
             int.TryParse(inconclusiveAttribute?.Value, out int inconclusive);
 
             var testResults = GetTestResults(testRunElement, _xmlns);
-            string trxOutput = Enumerable.Select<TestResult, string>(testResults, r => r.StdOut).Aggregate(new StringBuilder(), (acc, c) => acc.AppendLine(c)).ToString();
+            var leafTestResults =
+                testResults.Where(tr => tr.InnerResults.Count == 0)
+                           .Concat(testResults.Where(tr => tr.InnerResults.Count > 0).SelectMany(tr => tr.InnerResults))
+                           .ToArray();
+
+            string trxOutput = leafTestResults.Select(r => r.StdOut).Aggregate(new StringBuilder(), (acc, c) => acc.AppendLine(c)).ToString();
 
             return new TestExecutionResult
             {
                 ValidLicense = false,
                 TestResults = testResults,
+                LeafTestResults = leafTestResults,
                 Output = output,
                 ReportFiles = reportFiles.ToList(),
                 TrxOutput = trxOutput,
@@ -118,8 +126,12 @@ private TestExecutionResult CalculateMsTestTestExecutionResult(TestExecutionResu
         {
             bool HasPendingError(TestResult r) => r.ErrorMessage != null && r.ErrorMessage.Contains("Assert.Inconclusive failed. One or more step definitions are not implemented yet.");
 
-            testExecutionResult.Ignored = testExecutionResult.TestResults.Count(r => r.Outcome == "NotExecuted" && !HasPendingError(r));
-            testExecutionResult.Pending = testExecutionResult.TestResults.Count(r => HasPendingError(r));
+            testExecutionResult.Total = testExecutionResult.LeafTestResults.Length;
+            testExecutionResult.Succeeded = testExecutionResult.LeafTestResults.Count(tr => tr.Outcome == "Passed");
+            testExecutionResult.Failed = testExecutionResult.LeafTestResults.Count(tr => tr.Outcome == "Failed");
+            testExecutionResult.Executed = testExecutionResult.Succeeded + testExecutionResult.Failed;
+            testExecutionResult.Ignored = testExecutionResult.LeafTestResults.Count(r => r.Outcome == "NotExecuted" && !HasPendingError(r));
+            testExecutionResult.Pending = testExecutionResult.LeafTestResults.Count(HasPendingError);
 
             return testExecutionResult;
         }
@@ -135,21 +147,28 @@ private TestExecutionResult CalculateXUnitTestExecutionResult(TestExecutionResul
 
         private List<TestResult> GetTestResults(XElement testRunElement, XNamespace xmlns)
         {
-            var testResults = from unitTestResultElement in testRunElement.Element(xmlns + "Results")?.Elements(_unitTestResultElementName) ?? Enumerable.Empty<XElement>()
+            return GetTestResultsInternal(testRunElement.Element(xmlns + "Results"), xmlns);
+        }
+
+        private List<TestResult> GetTestResultsInternal(XElement testRunResultsElement, XNamespace xmlns)
+        {
+            var testResults = from unitTestResultElement in testRunResultsElement?.Elements(_unitTestResultElementName) ?? Enumerable.Empty<XElement>()
                 let outputElement = unitTestResultElement.Element(_unitTestResultOutputElementName)
                 let idAttribute = unitTestResultElement.Attribute("executionId")
                 let outcomeAttribute = unitTestResultElement.Attribute("outcome")
                 let stdOutElement = outputElement?.Element(_unitTestResultStdOutElementName)
                 let errorInfoElement = outputElement?.Element(xmlns + "ErrorInfo")
                 let errorMessage = errorInfoElement?.Element(xmlns + "Message")
+                let innerResultsElement = unitTestResultElement.Element(_unitTestResultInnerResultsElementName)
                 where idAttribute != null
                 where outcomeAttribute != null
                 select new TestResult
                 {
                     Id = idAttribute.Value,
                     Outcome = outcomeAttribute.Value,
                     StdOut = stdOutElement?.Value,
-                    ErrorMessage = errorMessage?.Value
+                    ErrorMessage = errorMessage?.Value,
+                    InnerResults = GetTestResultsInternal(innerResultsElement, xmlns)
                 };
 
             return testResults.ToList();

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TestExecutionResult.cs

@@ -15,6 +15,7 @@ public class TestExecutionResult
         public int Executed { get; set; }
 
         public List<TestResult> TestResults { get; set; }
+        public TestResult[] LeafTestResults { get; set; }
         public List<string> ReportFiles { get; set; }
         public int Warning { get; set; }
         public string LogFileContent { get; set; }
@@ -37,6 +38,7 @@ public class TestResult
         public List<TestStepResult> Steps { get; set; }
         public int ExecutionCount { get; set; }
         public string ErrorMessage { get; set; }
+        public List<TestResult> InnerResults { get; set; }
     }
 
     public class TestStepResult

Tests/Reqnroll.GeneratorTests/UnitTestProvider/MsTestV2GeneratorProviderTests.cs

@@ -2,6 +2,7 @@
 using Reqnroll.Generator.CodeDom;
 using Reqnroll.Generator.UnitTestProvider;
 using Reqnroll.Parser;
+using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
 using System.Linq;
@@ -11,7 +12,10 @@ namespace Reqnroll.GeneratorTests.UnitTestProvider
 {
     public class MsTestV2GeneratorProviderTests
     {
-        private const string TestDeploymentItemAttributeName = "Microsoft.VisualStudio.TestTools.UnitTesting.DeploymentItemAttribute";
+        private const string 
+            TestCaseAttributeName = "Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute",
+            TestDeploymentItemAttributeName = "Microsoft.VisualStudio.TestTools.UnitTesting.DeploymentItemAttribute",
+            TestMethodAttributeName = "Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute";
 
         [Fact]
         public void MsTestV2GeneratorProvider_WithFeatureWithoutDeploymentItem_GeneratedClassShouldNotIncludeDeploymentItemForPlugin()
@@ -164,6 +168,71 @@ public void MsTestV2GeneratorProvider_WithoutPriorityTag_ShouldntAddPriorityAttr
             testMethod.CustomAttributes().Should().NotContain(a => a.Name == MsTestV2GeneratorProvider.PRIORITY_ATTR);
         }
 
+        [Fact]
+        public void MsTestV2GeneratorProvider_WithScenarioOutline_ShouldGenerateASingleParametrizedTest()
+        {
+            // ARRANGE
+            var document = ParseDocumentFromString(@"
+            Feature: Sample feature file
+
+            Scenario Outline: Simple scenario
+                Given there is <count> <color> items
+
+                Examples:
+                    | count | color |
+                    |     0 | blue  |
+                    |     2 | black |");
+
+            var provider = new MsTestV2GeneratorProvider(new CodeDomHelper(CodeDomProviderLanguage.CSharp));
+            var featureGenerator = provider.CreateFeatureGenerator();
+
+            // ACT
+            var code = featureGenerator.GenerateUnitTestFixture(document, "TestClassName", "Target.Namespace");
+
+            // ASSERT
+            var testMethods = code.Class().Members().Where(x => x.CustomAttributes().Any(a => a.Name == TestMethodAttributeName)).ToList();
+            testMethods.Should().HaveCount(1);
+            var testCaseAttributes = testMethods[0].CustomAttributes().Where(x => x.Name == TestCaseAttributeName).ToList();
+            testCaseAttributes.Should().HaveCount(2);
+            testCaseAttributes.Should().ContainSingle(x => x.ArgumentValues().OfType<string>().ElementAt(0) == "0" &&
+                                                           x.ArgumentValues().OfType<string>().ElementAt(1) == "blue");
+            testCaseAttributes.Should().ContainSingle(x => x.ArgumentValues().OfType<string>().ElementAt(0) == "2" &&
+                                                           x.ArgumentValues().OfType<string>().ElementAt(1) == "black");
+        }
+
+        [Fact]
+        public void MsTestV2GeneratorProvider_WithScenarioOutline_ShouldSkipIgnoredExamples()
+        {
+            // ARRANGE
+            var document = ParseDocumentFromString(@"
+            Feature: Sample feature file
+
+            Scenario Outline: Simple scenario
+                Given there is <count> items
+
+                @ignore
+                Examples:
+                    | count |
+                    |     0 |
+
+                Examples:
+                    | count |
+                    |     2 |");
+
+            var provider = new MsTestV2GeneratorProvider(new CodeDomHelper(CodeDomProviderLanguage.CSharp));
+            var featureGenerator = provider.CreateFeatureGenerator();
+
+            // ACT
+            var code = featureGenerator.GenerateUnitTestFixture(document, "TestClassName", "Target.Namespace");
+
+            // ASSERT
+            var testMethods = code.Class().Members().Where(x => x.CustomAttributes().Any(a => a.Name == TestMethodAttributeName)).ToList();
+            testMethods.Should().HaveCount(1);
+            var testCaseAttributes = testMethods[0].CustomAttributes().Where(x => x.Name == TestCaseAttributeName).ToList();
+            testCaseAttributes.Should().HaveCount(1);
+            testCaseAttributes.Should().ContainSingle(x => x.ArgumentValues().OfType<string>().ElementAt(0) == "2");
+        }
+
         public ReqnrollDocument ParseDocumentFromString(string documentSource, CultureInfo parserCultureInfo = null)
         {
             var parser = new ReqnrollGherkinParser(parserCultureInfo ?? new CultureInfo("en-US"));

Tests/Reqnroll.Specs/Features/RegressionTests/MsTest/GH1052.feature

@@ -1,7 +1,7 @@
 @MSTest
 Feature: GH1052
 
-Wrong scenario context injected when running tests in parallel using NUnit (also specrun) - https://github.com/reqnroll/Reqnroll/issues/1052
+Wrong scenario context injected when running tests in parallel using NUnit (also specrun) - https://github.com/SpecFlowOSS/SpecFlow/issues/1052
 
 
 Scenario: GH1052
@@ -133,7 +133,7 @@ Scenario: GH1052
                 {
                     _reqnrollOutputHelper.WriteLine($"Context ID {_scenarioContext["ID"].ToString()}");
                     _reqnrollOutputHelper.WriteLine($"ManagedThreadId {Thread.CurrentThread.ManagedThreadId}");
-                    Thread.Sleep(1000);
+                    Thread.Sleep(100);
                 }
                 catch (Exception e)
                 {

Tests/Reqnroll.Specs/StepDefinitions/ExecutionResultSteps.cs

@@ -108,7 +108,7 @@ public void ThenEveryScenarioHasItSIndividualContextId()
         {
             var lastTestExecutionResult = _vsTestExecutionDriver.LastTestExecutionResult;
 
-            foreach (var testResult in lastTestExecutionResult.TestResults)
+            foreach (var testResult in lastTestExecutionResult.LeafTestResults)
             {
                 var contextIdLines = testResult.StdOut.Split(new string[] { Environment.NewLine, "\n" }, StringSplitOptions.RemoveEmptyEntries).Where(s => s.StartsWith("-> Context ID"));