From 45f6d9c5280c2cede7580a1a9f29029bebd6efec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Wed, 27 Mar 2024 18:19:31 +0100 Subject: [PATCH] Introduce System tests (#82) * system tests concept # Conflicts: # Reqnroll.sln * Update test for problem identified for #54 by @livioc * replace BoDi with Reqnroll.BoDi * portability test POC * Undo 'Microsoft.Build.Locator' for MsBuild location and make it flexible with env vars (#75) * add portability compilation tests with msbuild * Add system tests to CI * Fix system tests to CI * Add TODOs for further portability tests * Fix system tests to CI, take 2 * Change SystemTests to use MsTest * set default target framework to .NET 8.0 * Add .NET Information to CI * test specs filter on CI * Revert "test specs filter on CI" This reverts commit bdcdbaa9fbdca5e16e7b96686ee65f5fa61bd414. * fix specs_filter on CI * speed up build by using heuristics to find MsBuild * speed up build by using global nuget packages folder * cleanup CI script * add CI system test for linux * create Generation test structure * add more categories and exclude .NET Framework and MsBuild tests from linux * Add NUnit and xUnit tests * rename AppConfigDriver to ConfigurationDriver, allow customizing nuget global folder * fix strange test error if .NET 4.6.2 test target was not included in run * Add smoke test * TEST: remove test filter from Linux CI * Add Generation test TODO comments * re-add filter to linux CI * enable parallel system testing * set target fw of Reqnroll.TestProjectGenerator to netstandard2.0 * re-enable all specs tests * code cleanup --- .github/workflows/ci.yml | 68 +++++++- ...oll.TestProjectGenerator.v3.ncrunchproject | 6 +- .../NuGetConfigGeneratorTests.cs | 1 - .../ProjectTests.cs | 84 ++++++++-- .../AppConfigDriver.cs | 10 -- .../Reqnroll.TestProjectGenerator/Compiler.cs | 10 +- .../ConfigurationDriver.cs | 29 ++++ .../Data/TargetFramework.cs | 8 +- ...CommandBuilder.NewProjectCommandBuilder.cs | 7 +- .../Driver/SolutionDriver.cs | 16 +- .../ProgrammingLanguageExtensions.cs | 4 +- .../BindingsGeneratorFactory.cs | 4 +- .../CSharpBindingsGenerator.cs | 1 - .../Factories/ProjectBuilderFactory.cs | 16 +- .../NetCoreSdkInfoProvider.cs | 31 ++-- .../OldFormatProjectWriter.cs | 2 +- .../FilesystemWriter/SolutionWriter.cs | 18 +-- .../Reqnroll.TestProjectGenerator/Folders.cs | 13 +- .../MSBuildFinder.cs | 74 +++++++-- .../Reqnroll.TestProjectGenerator/NuGet.cs | 2 +- .../ProgrammingLanguage.cs | 4 +- .../ProjectBuilder.cs | 6 +- .../Reqnroll.TestProjectGenerator.csproj | 3 +- .../TargetFrameworkMonikerStringBuilder.cs | 8 +- Reqnroll.sln | 15 ++ .../TestRunCombinations.cs | 14 +- .../ReqnrollConfigurationSteps.cs | 1 + .../AssemblyAttributes.cs | 3 + .../Drivers/ConsoleOutputConnector.cs | 31 ++++ .../Drivers/ExecutionDriver.cs | 38 +++++ .../Drivers/TestFileManager.cs | 38 +++++ .../Generation/GenerationTestBase.cs | 24 +++ .../Generation/MsTestGenerationTest.cs | 47 ++++++ .../Generation/NUnitGenerationTest.cs | 19 +++ .../Generation/XUnitGenerationTest.cs | 19 +++ .../NuGetPackageVersion.cs | 6 + .../NuGetPackageVersion.template.cs.txt | 6 + .../Portability/Net462PortabilityTest.cs | 18 +++ .../Portability/Net472PortabilityTest.cs | 18 +++ .../Portability/Net481PortabilityTest.cs | 18 +++ .../Portability/Net60PortabilityTest.cs | 16 ++ .../Portability/Net70PortabilityTest.cs | 16 ++ .../Portability/Net80PortabilityTest.cs | 16 ++ .../Portability/PortabilityTestBase.cs | 46 ++++++ .../Reqnroll.SystemTests.csproj | 51 ++++++ .../Resources/GeneratorAllInSample1.feature | 81 ++++++++++ .../Resources/GeneratorAllInSample2.feature | 40 +++++ Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs | 23 +++ Tests/Reqnroll.SystemTests/SystemTestBase.cs | 149 ++++++++++++++++++ 49 files changed, 1063 insertions(+), 115 deletions(-) delete mode 100644 Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/AppConfigDriver.cs create mode 100644 Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ConfigurationDriver.cs create mode 100644 Tests/Reqnroll.SystemTests/AssemblyAttributes.cs create mode 100644 Tests/Reqnroll.SystemTests/Drivers/ConsoleOutputConnector.cs create mode 100644 Tests/Reqnroll.SystemTests/Drivers/ExecutionDriver.cs create mode 100644 Tests/Reqnroll.SystemTests/Drivers/TestFileManager.cs create mode 100644 Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs create mode 100644 Tests/Reqnroll.SystemTests/Generation/MsTestGenerationTest.cs create mode 100644 Tests/Reqnroll.SystemTests/Generation/NUnitGenerationTest.cs create mode 100644 Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs create mode 100644 Tests/Reqnroll.SystemTests/NuGetPackageVersion.cs create mode 100644 Tests/Reqnroll.SystemTests/NuGetPackageVersion.template.cs.txt create mode 100644 Tests/Reqnroll.SystemTests/Portability/Net462PortabilityTest.cs create mode 100644 Tests/Reqnroll.SystemTests/Portability/Net472PortabilityTest.cs create mode 100644 Tests/Reqnroll.SystemTests/Portability/Net481PortabilityTest.cs create mode 100644 Tests/Reqnroll.SystemTests/Portability/Net60PortabilityTest.cs create mode 100644 Tests/Reqnroll.SystemTests/Portability/Net70PortabilityTest.cs create mode 100644 Tests/Reqnroll.SystemTests/Portability/Net80PortabilityTest.cs create mode 100644 Tests/Reqnroll.SystemTests/Portability/PortabilityTestBase.cs create mode 100644 Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj create mode 100644 Tests/Reqnroll.SystemTests/Resources/GeneratorAllInSample1.feature create mode 100644 Tests/Reqnroll.SystemTests/Resources/GeneratorAllInSample2.feature create mode 100644 Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs create mode 100644 Tests/Reqnroll.SystemTests/SystemTestBase.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c86da297..c782b131d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ permissions: env: SPECS_FILTER: "" # use for testing CI: "&Category=basicExecution" + REQNROLL_TEST_PIPELINEMODE: true jobs: build: @@ -48,6 +49,7 @@ jobs: product_configuration: ${{ steps.versions.outputs.product_configuration }} build_params: ${{ steps.versions.outputs.build_params }} test_params: ${{ steps.versions.outputs.test_params }} + specs_filter: ${{ steps.versions.outputs.specs_filter }} steps: - uses: actions/checkout@v4 @@ -97,7 +99,7 @@ jobs: $specsFilter = "&$specsFilter" } else { - $specsFilter = $envSPECS_FILTER + $specsFilter = $env:SPECS_FILTER } Write-Output "specs_filter=$specsFilter" >> $env:GITHUB_OUTPUT Write-Output "Specs Filter: $specsFilter" @@ -248,3 +250,67 @@ jobs: with: name: specs-mstest-trx path: "**/specs-mstest-results.trx" + + system-tests-windows: + runs-on: windows-latest + needs: build + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # avoid shallow clone so nbgv can do its work. + - name: .NET Information + run: | + dotnet --list-sdks + dotnet --list-runtimes + - name: Restore dependencies + run: dotnet restore + - name: Install Test Report Dependencies + run: | + dotnet add ./Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj package GitHubActionsTestLogger + - name: Build + run: dotnet build --no-restore ${{ needs.build.outputs.build_params }} + - name: System Tests + shell: pwsh + run: | + $gitHubActionsLoggerSettings = '"GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true;annotations.titleFormat=[@traits.Category] @test;annotations.messageFormat=@error\n@trace"' + dotnet test ./Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj ${{ needs.build.outputs.test_params }} --logger "trx;LogFileName=systemtests-windows-results.trx" --logger $gitHubActionsLoggerSettings -- RunConfiguration.CollectSourceInformation=true + - name: Upload Test Result TRX Files + uses: actions/upload-artifact@v4 + if: always() + with: + name: systemtests-windows-trx + path: "**/*.trx" + + + system-tests-linux: + runs-on: ubuntu-latest + needs: build + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # avoid shallow clone so nbgv can do its work. + - name: .NET Information + run: | + dotnet --list-sdks + dotnet --list-runtimes + - name: Restore dependencies + run: dotnet restore + - name: Install Test Report Dependencies + run: | + dotnet add ./Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj package GitHubActionsTestLogger + - name: Build + run: dotnet build --no-restore ${{ needs.build.outputs.build_params }} + - name: System Tests + shell: pwsh + run: | + $gitHubActionsLoggerSettings = '"GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true;annotations.titleFormat=[@traits.Category] @test;annotations.messageFormat=@error\n@trace"' + dotnet test ./Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj ${{ needs.build.outputs.test_params }} --filter "TestCategory!=MsBuild&TestCategory!=Net481" --logger "trx;LogFileName=systemtests-linux-results.trx" --logger $gitHubActionsLoggerSettings -- RunConfiguration.CollectSourceInformation=true + - name: Upload Test Result TRX Files + uses: actions/upload-artifact@v4 + if: always() + with: + name: systemtests-linux-trx + path: "**/*.trx" + \ No newline at end of file diff --git a/.ncrunch/Reqnroll.TestProjectGenerator.v3.ncrunchproject b/.ncrunch/Reqnroll.TestProjectGenerator.v3.ncrunchproject index e091aa0f0..95a483b43 100644 --- a/.ncrunch/Reqnroll.TestProjectGenerator.v3.ncrunchproject +++ b/.ncrunch/Reqnroll.TestProjectGenerator.v3.ncrunchproject @@ -1,7 +1,3 @@  - - - TargetFrameworks = net60 - - + \ No newline at end of file diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/NuGetConfigGeneratorTests.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/NuGetConfigGeneratorTests.cs index 5a410d8ef..efcec2988 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/NuGetConfigGeneratorTests.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/NuGetConfigGeneratorTests.cs @@ -1,6 +1,5 @@ using FluentAssertions; using Reqnroll.TestProjectGenerator.Data; -using Reqnroll.TestProjectGenerator.NewApi._1_Memory; using Xunit; namespace Reqnroll.TestProjectGenerator.Tests diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/ProjectTests.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/ProjectTests.cs index 5f6095b2f..1797b9360 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/ProjectTests.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/ProjectTests.cs @@ -238,7 +238,7 @@ private string GetProjectFolderPath(string solutionFolder, Project project) } [Fact] - public void CreateEmtpyCSharpProjectInNewFormat() + public void CreateEmptyCSharpProjectInNewFormat() { var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp); @@ -246,12 +246,11 @@ public void CreateEmtpyCSharpProjectInNewFormat() string projectFileContent = GetProjectFileContent(solutionFolder, project); - projectFileContent.Should() - .Contain("\r\n \r\n net462\r\n \r\n"); + projectFileContent.Should().Contain(""); } [Fact] - public void CreateEmtpyCSharpCore3_1ProjectInNewFormat() + public void CreateEmptyCSharpCore3_1ProjectInNewFormat() { var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Netcoreapp31); @@ -264,20 +263,20 @@ public void CreateEmtpyCSharpCore3_1ProjectInNewFormat() } [Fact] - public void CreateEmtpyCSharpNet50ProjectInNewFormat() + public void CreateEmptyCSharpNet50ProjectInNewFormat() { - var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Net50); + var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp73, TargetFramework.Net50); new SolutionWriter(new Mock().Object).WriteToFileSystem(solution, solutionFolder); string projectFileContent = GetProjectFileContent(solutionFolder, project); projectFileContent.Should() - .Contain("\r\n \r\n net5.0\r\n \r\n"); + .Contain("\r\n \r\n net5.0\r\n 7.3\r\n \r\n"); } [Fact] - public void CreateEmtpyCSharpNet60ProjectInNewFormat() + public void CreateEmptyCSharpNet60ProjectInNewFormat() { var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Net60); @@ -290,7 +289,72 @@ public void CreateEmtpyCSharpNet60ProjectInNewFormat() } [Fact] - public void CreateEmtpyFSharpProjectInNewFormat() + public void CreateEmptyCSharpNet70ProjectInNewFormat() + { + var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Net70); + + new SolutionWriter(new Mock().Object).WriteToFileSystem(solution, solutionFolder); + + string projectFileContent = GetProjectFileContent(solutionFolder, project); + + projectFileContent.Should() + .Contain("\r\n \r\n net7.0\r\n enable\r\n enable\r\n \r\n"); + } + + [Fact] + public void CreateEmptyCSharpNet80ProjectInNewFormat() + { + var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Net80); + + new SolutionWriter(new Mock().Object).WriteToFileSystem(solution, solutionFolder); + + string projectFileContent = GetProjectFileContent(solutionFolder, project); + + projectFileContent.Should() + .Contain("\r\n \r\n net8.0\r\n enable\r\n enable\r\n \r\n"); + } + + [Fact] + public void CreateEmptyCSharpNet481ProjectInNewFormat() + { + var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp73, TargetFramework.Net481); + + new SolutionWriter(new Mock().Object).WriteToFileSystem(solution, solutionFolder); + + string projectFileContent = GetProjectFileContent(solutionFolder, project); + + projectFileContent.Should() + .Contain("\r\n \r\n net481\r\n 7.3\r\n \r\n"); + } + + [Fact] + public void CreateEmptyCSharpNet462ProjectInNewFormat() + { + var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp73, TargetFramework.Net462); + + new SolutionWriter(new Mock().Object).WriteToFileSystem(solution, solutionFolder); + + string projectFileContent = GetProjectFileContent(solutionFolder, project); + + projectFileContent.Should() + .Contain("\r\n \r\n net462\r\n 7.3\r\n \r\n"); + } + + [Fact] + public void CreateEmptyCSharpNet472ProjectInNewFormat() + { + var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp73, TargetFramework.Net472); + + new SolutionWriter(new Mock().Object).WriteToFileSystem(solution, solutionFolder); + + string projectFileContent = GetProjectFileContent(solutionFolder, project); + + projectFileContent.Should() + .Contain("\r\n \r\n net472\r\n 7.3\r\n \r\n"); + } + + [Fact] + public void CreateEmptyFSharpProjectInNewFormat() { var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.FSharp); @@ -303,7 +367,7 @@ public void CreateEmtpyFSharpProjectInNewFormat() } [Fact] - public void CreateEmtpyVbProjectInNewFormat() + public void CreateEmptyVbProjectInNewFormat() { var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.VB); diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/AppConfigDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/AppConfigDriver.cs deleted file mode 100644 index 5446c9fc7..000000000 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/AppConfigDriver.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System.Configuration; - -namespace Reqnroll.TestProjectGenerator -{ - public class AppConfigDriver - { - public string TestProjectFolderName => ConfigurationManager.AppSettings["testProjectFolder"] ?? "RR"; - public string VSTestPath => ConfigurationManager.AppSettings["vstestPath"] ?? "Common7\\IDE\\CommonExtensions\\Microsoft\\TestWindow"; - } -} \ No newline at end of file diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Compiler.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Compiler.cs index 443722303..f8a286710 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Compiler.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Compiler.cs @@ -34,15 +34,7 @@ public CompileResult Run(BuildTool buildTool, bool? treatWarningsAsErrors) private CompileResult CompileWithMSBuild(bool? treatWarningsAsErrors) { - string msBuildPath=""; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - msBuildPath = _msBuildFinder.FindMSBuild(); - } - else - { - msBuildPath = "msbuild"; - } + var msBuildPath = _msBuildFinder.FindMSBuild(); _outputWriter.WriteLine($"Invoke MsBuild from {msBuildPath}"); diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ConfigurationDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ConfigurationDriver.cs new file mode 100644 index 000000000..04182607d --- /dev/null +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ConfigurationDriver.cs @@ -0,0 +1,29 @@ +using System; +using System.Configuration; + +namespace Reqnroll.TestProjectGenerator; + +public class ConfigurationDriver +{ + public string TestProjectFolderName => GetConfigSetting("testProjectFolder", "RR"); + public string VSTestPath => GetConfigSetting("vstestPath", "Common7\\IDE\\CommonExtensions\\Microsoft\\TestWindow"); + public string MsBuildPath => GetConfigSetting(nameof(MsBuildPath)); + public bool PipelineMode => GetConfigSwitch(nameof(PipelineMode)); + public string GlobalNuGetPackages => GetConfigSetting(nameof(GlobalNuGetPackages)); + + public bool GetConfigSwitch(string key, bool defaultValue = false) + { + return GetConfigSetting(key, defaultValue.ToString()).Equals("true", StringComparison.InvariantCultureIgnoreCase); + } + + public string GetConfigSetting(string key, string defaultValue = null) + { + var envSetting = Environment.GetEnvironmentVariable($"REQNROLL_TEST_{key}".ToUpperInvariant()); + if (!string.IsNullOrEmpty(envSetting)) return envSetting; + + var configSetting = ConfigurationManager.AppSettings[key]; + if (!string.IsNullOrEmpty(configSetting)) return configSetting; + + return defaultValue; + } +} \ No newline at end of file diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/TargetFramework.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/TargetFramework.cs index 56e83c7fe..b1e5ef73b 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/TargetFramework.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/TargetFramework.cs @@ -13,7 +13,13 @@ public enum TargetFramework Netcoreapp31, Net461, Net462, + Net471, + Net472, + Net48, + Net481, Net50, - Net60 + Net60, + Net70, + Net80 } } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Dotnet/NewCommandBuilder.NewProjectCommandBuilder.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Dotnet/NewCommandBuilder.NewProjectCommandBuilder.cs index 90cf22adb..fd6d6df2c 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Dotnet/NewCommandBuilder.NewProjectCommandBuilder.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Dotnet/NewCommandBuilder.NewProjectCommandBuilder.cs @@ -51,11 +51,16 @@ protected override string BuildArguments() arguments = AddArgument( arguments, "-lang", + _language == ProgrammingLanguage.CSharp73 ? "\"C#\"" : _language == ProgrammingLanguage.CSharp ? "\"C#\"" : - _language == ProgrammingLanguage.CSharp10 ? "\"C#\"" : _language == ProgrammingLanguage.VB ? "VB" : _language == ProgrammingLanguage.FSharp ? "\"F#\"" : string.Empty); + if (_language == ProgrammingLanguage.CSharp73) + { + arguments = AddArgument(arguments, "--langVersion", "7.3"); + } + return arguments; } } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/SolutionDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/SolutionDriver.cs index 4314001ee..159a5a31f 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/SolutionDriver.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/SolutionDriver.cs @@ -17,6 +17,7 @@ public class SolutionDriver private readonly ProjectBuilderFactory _projectBuilderFactory; private readonly Folders _folders; private readonly ArtifactNamingConvention _artifactNamingConvention; + private readonly IOutputWriter _outputWriter; private readonly Solution _solution; private readonly Dictionary _projects = new Dictionary(); private ProjectBuilder _defaultProject; @@ -27,18 +28,20 @@ public SolutionDriver( ProjectBuilderFactory projectBuilderFactory, Folders folders, TestProjectFolders testProjectFolders, - ArtifactNamingConvention artifactNamingConvention) + ArtifactNamingConvention artifactNamingConvention, + IOutputWriter outputWriter) { _nuGetConfigGenerator = nuGetConfigGenerator; _testRunConfiguration = testRunConfiguration; _projectBuilderFactory = projectBuilderFactory; _folders = folders; _artifactNamingConvention = artifactNamingConvention; + _outputWriter = outputWriter; NuGetSources = new List { - new NuGetSource("LocalReqnrollDevPackages", _folders.NuGetFolder), - new NuGetSource("Reqnroll CI", "https://www.myget.org/F/reqnroll/api/v3/index.json"), - new NuGetSource("Reqnroll Unstable", "https://www.myget.org/F/reqnroll-unstable/api/v3/index.json") + new("LocalReqnrollDevPackages", _folders.NuGetFolder), + new("Reqnroll CI", "https://www.myget.org/F/reqnroll/api/v3/index.json"), + new("Reqnroll Unstable", "https://www.myget.org/F/reqnroll-unstable/api/v3/index.json") }; if (testRunConfiguration.UnitTestProvider == UnitTestProvider.SpecRun) @@ -86,7 +89,10 @@ public Solution GetSolution() _solution.AddProject(project); } - _solution.NugetConfig = _nuGetConfigGenerator?.Generate(NuGetSources.ToArray(), _folders.RunUniqueGlobalPackages); + var customGlobalPackagesFolder = _folders.IsGlobalPackagesCustomized ? _folders.GlobalNuGetPackages : null; + if (customGlobalPackagesFolder != null) + _outputWriter.WriteLine($"Using custom global packages folder: {customGlobalPackagesFolder}"); + _solution.NugetConfig = _nuGetConfigGenerator?.Generate(NuGetSources.ToArray(), customGlobalPackagesFolder); return _solution; } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Extensions/ProgrammingLanguageExtensions.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Extensions/ProgrammingLanguageExtensions.cs index 7c5bc56a7..7bba17b2e 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Extensions/ProgrammingLanguageExtensions.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Extensions/ProgrammingLanguageExtensions.cs @@ -9,8 +9,8 @@ public static string ToProjectFileExtension(this ProgrammingLanguage programming { switch (programmingLanguage) { + case ProgrammingLanguage.CSharp73: return "csproj"; case ProgrammingLanguage.CSharp: return "csproj"; - case ProgrammingLanguage.CSharp10: return "csproj"; case ProgrammingLanguage.FSharp: return "fsproj"; case ProgrammingLanguage.VB: return "vbproj"; default: @@ -23,8 +23,8 @@ public static string ToCodeFileExtension(this ProgrammingLanguage programmingLan { switch (programmingLanguage) { + case ProgrammingLanguage.CSharp73: return "cs"; case ProgrammingLanguage.CSharp: return "cs"; - case ProgrammingLanguage.CSharp10: return "cs"; case ProgrammingLanguage.FSharp: return "fs"; case ProgrammingLanguage.VB: return "vb"; default: throw new NotSupportedException("There is no known file extension for the programming language."); diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BindingsGeneratorFactory.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BindingsGeneratorFactory.cs index ce959ea42..47b14ea22 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BindingsGeneratorFactory.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BindingsGeneratorFactory.cs @@ -8,8 +8,8 @@ public BaseBindingsGenerator FromLanguage(ProgrammingLanguage targetLanguage) { switch (targetLanguage) { - case ProgrammingLanguage.CSharp: return new CSharpBindingsGenerator(); - case ProgrammingLanguage.CSharp10: return new CSharp10BindingsGenerator(); + case ProgrammingLanguage.CSharp73: return new CSharpBindingsGenerator(); + case ProgrammingLanguage.CSharp: return new CSharp10BindingsGenerator(); case ProgrammingLanguage.FSharp: return new FSharpBindingsGenerator(); case ProgrammingLanguage.VB: return new VbBindingsGenerator(); default: throw new ArgumentException( diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs index f068b9008..fd0a17861 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs @@ -64,7 +64,6 @@ public override ProjectFile GenerateStepDefinition(string method) public override ProjectFile GenerateLoggerClass(string pathToLogFile) { string fileContent = $$""" - #nullable disable using System; using System.IO; using System.Runtime.CompilerServices; diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ProjectBuilderFactory.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ProjectBuilderFactory.cs index da340d9de..54437d9ec 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ProjectBuilderFactory.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ProjectBuilderFactory.cs @@ -72,20 +72,22 @@ public ProgrammingLanguage ParseCSharpProgrammingLanguage() { switch (_testRunConfiguration.TargetFramework) { + case TargetFramework.Net35: case TargetFramework.Net45: - case TargetFramework.NetStandard20: - case TargetFramework.Netcoreapp20: case TargetFramework.Net452: - case TargetFramework.Net35: + case TargetFramework.Net461: + case TargetFramework.Net462: + case TargetFramework.Net471: + case TargetFramework.Net472: + case TargetFramework.Netcoreapp20: case TargetFramework.Netcoreapp21: case TargetFramework.Netcoreapp22: case TargetFramework.Netcoreapp30: case TargetFramework.Netcoreapp31: - case TargetFramework.Net461: - case TargetFramework.Net462: case TargetFramework.Net50: - return ProgrammingLanguage.CSharp; - default: return ProgrammingLanguage.CSharp10; + case TargetFramework.NetStandard20: + return ProgrammingLanguage.CSharp73; + default: return ProgrammingLanguage.CSharp; } } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/NetCoreSdkInfoProvider.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/NetCoreSdkInfoProvider.cs index ff855a488..1c6dcca27 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/NetCoreSdkInfoProvider.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/NetCoreSdkInfoProvider.cs @@ -5,28 +5,35 @@ namespace Reqnroll.TestProjectGenerator.FilesystemWriter { public class NetCoreSdkInfoProvider { - private static readonly NetCoreSdkInfo NetCore31 = new NetCoreSdkInfo("3.1.201"); - private static readonly NetCoreSdkInfo NetCore30 = new NetCoreSdkInfo("3.0.101"); - private static readonly NetCoreSdkInfo NetCore22 = new NetCoreSdkInfo("2.2.402"); - private static readonly NetCoreSdkInfo NetCore21 = new NetCoreSdkInfo("2.1.807"); - private static readonly NetCoreSdkInfo NetCore20 = new NetCoreSdkInfo("2.1.202"); - private static readonly NetCoreSdkInfo Net461 = new NetCoreSdkInfo("3.1.201"); - private static readonly NetCoreSdkInfo Net462 = new NetCoreSdkInfo("3.1.201"); - private static readonly NetCoreSdkInfo Net50 = new NetCoreSdkInfo("5.0.404"); - private static readonly NetCoreSdkInfo Net60 = new NetCoreSdkInfo("6.0.100"); + private static readonly NetCoreSdkInfo DefaultSdk = new("6.0.100"); + private static readonly NetCoreSdkInfo NetCore31 = new("3.1.201"); + private static readonly NetCoreSdkInfo NetCore30 = new("3.0.101"); + private static readonly NetCoreSdkInfo NetCore22 = new("2.2.402"); + private static readonly NetCoreSdkInfo NetCore21 = new("2.1.807"); + private static readonly NetCoreSdkInfo NetCore20 = new("2.1.202"); + private static readonly NetCoreSdkInfo Net50 = new("5.0.404"); + private static readonly NetCoreSdkInfo Net60 = new("6.0.100"); + private static readonly NetCoreSdkInfo Net70 = new("7.0.100"); + private static readonly NetCoreSdkInfo Net80 = new("8.0.100"); private readonly IReadOnlyDictionary _sdkMappings = new Dictionary { - [TargetFramework.Net461] = Net461, - [TargetFramework.Net462] = Net462, + [TargetFramework.Net461] = DefaultSdk, + [TargetFramework.Net462] = DefaultSdk, + [TargetFramework.Net471] = DefaultSdk, + [TargetFramework.Net472] = DefaultSdk, + [TargetFramework.Net48] = DefaultSdk, + [TargetFramework.Net481] = DefaultSdk, [TargetFramework.Net50] = Net50, [TargetFramework.Net60] = Net60, + [TargetFramework.Net70] = Net70, + [TargetFramework.Net80] = Net80, [TargetFramework.Netcoreapp31] = NetCore31, [TargetFramework.Netcoreapp30] = NetCore30, [TargetFramework.Netcoreapp22] = NetCore22, [TargetFramework.Netcoreapp21] = NetCore21, [TargetFramework.Netcoreapp20] = NetCore20, - [TargetFramework.NetStandard20] = NetCore22 + [TargetFramework.NetStandard20] = DefaultSdk }; public NetCoreSdkInfo GetSdkFromTargetFramework(TargetFramework targetFramework) diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/OldFormatProjectWriter.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/OldFormatProjectWriter.cs index 18ba9bc13..bf783ffff 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/OldFormatProjectWriter.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/OldFormatProjectWriter.cs @@ -96,8 +96,8 @@ private void WriteLanguageTargets(XmlWriter xw, Project project) { switch (project.ProgrammingLanguage) { + case ProgrammingLanguage.CSharp73: case ProgrammingLanguage.CSharp: - case ProgrammingLanguage.CSharp10: WriteMSBuildImport(xw, "$(MSBuildToolsPath)\\Microsoft.CSharp.targets"); break; case ProgrammingLanguage.FSharp: diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/SolutionWriter.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/SolutionWriter.cs index 4756924e2..34e646b4c 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/SolutionWriter.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/SolutionWriter.cs @@ -26,7 +26,7 @@ public string WriteToFileSystem(Solution solution, string outputPath) { if (!Directory.Exists(outputPath)) { - Directory.CreateDirectory(outputPath); + Directory.CreateDirectory(outputPath!); } if (solution is null) @@ -51,7 +51,7 @@ public string WriteToFileSystem(Solution solution, string outputPath) _fileWriter.Write(globalJsonFile, outputPath); } - AllowNet6ToTestOlderFrameworks(targetFramework); + DisableUsingSdkFromEnvironmentVariable(); var createSolutionCommand = DotNet.New(_outputWriter).Solution().InFolder(outputPath).WithName(solution.Name).Build(); createSolutionCommand.ExecuteWithRetry(1, TimeSpan.FromSeconds(1), @@ -74,15 +74,15 @@ public string WriteToFileSystem(Solution solution, string outputPath) } /// - /// (santa) 2021.10.28 In .Net6 prerelease there is an issue when using 'dotnet' commands from tests - /// The environment variable is set to MSBuildSDKsPath=C:\Program Files\dotnet\sdk\6.0.100-rc.1.21463.6\Sdks - /// As a result the dotnet restore command fails - /// The workaround is to remove this environment variable + /// During test execution, the MSBuildSDKsPath environment variable is set to the SDK of the test execution, + /// e.g. C:\Program Files\dotnet\sdk\8.0.101\Sdks. + /// This causes issues with dotnet restore if working with a project of a different target framework. + /// The error might be for example 'error MSB4062: The "CheckIfPackageReferenceShouldBeFrameworkReference" task could not be loaded from the assembly[...]'. + /// The workaround is to remove this environment variable. /// - /// - private static void AllowNet6ToTestOlderFrameworks(TargetFramework targetFramework) + private static void DisableUsingSdkFromEnvironmentVariable() { - if (targetFramework is TargetFramework.Net461 or TargetFramework.Netcoreapp31 or TargetFramework.Netcoreapp21 or TargetFramework.Net50 or TargetFramework.Net462) + if (Environment.GetEnvironmentVariable("MSBuildSDKsPath") != null) Environment.SetEnvironmentVariable("MSBuildSDKsPath", null); } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Folders.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Folders.cs index 01a3f14ee..ab12317cd 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Folders.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Folders.cs @@ -8,7 +8,7 @@ namespace Reqnroll.TestProjectGenerator { public class Folders { - private readonly AppConfigDriver _appConfigDriver; + private readonly ConfigurationDriver _configurationDriver; private readonly ArtifactNamingConvention _artifactNamingConvention; private static readonly Guid UniqueRunId = Guid.NewGuid(); protected string _nugetFolder; @@ -22,9 +22,9 @@ public class Folders public string TestFolder => Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().Location).LocalPath); - public Folders(AppConfigDriver appConfigDriver, ArtifactNamingConvention artifactNamingConvention) + public Folders(ConfigurationDriver configurationDriver, ArtifactNamingConvention artifactNamingConvention) { - _appConfigDriver = appConfigDriver; + _configurationDriver = configurationDriver; _artifactNamingConvention = artifactNamingConvention; } @@ -69,7 +69,7 @@ public string PackageFolder set => _packageFolder = value; } - public string GlobalPackages => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages"); + public string SystemGlobalNuGetPackages => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages"); public string Reqnroll { @@ -77,9 +77,10 @@ public string Reqnroll set => _reqnroll = value; } - public virtual string FolderToSaveGeneratedSolutions => Path.Combine(Path.GetTempPath(), _appConfigDriver.TestProjectFolderName); + public virtual string FolderToSaveGeneratedSolutions => Path.Combine(Path.GetTempPath(), _configurationDriver.TestProjectFolderName); public virtual string RunUniqueFolderToSaveGeneratedSolutions => Path.Combine(FolderToSaveGeneratedSolutions, _artifactNamingConvention.GetRunName(UniqueRunId)); - public virtual string RunUniqueGlobalPackages => Path.Combine(RunUniqueFolderToSaveGeneratedSolutions, ".nuget"); + public virtual string GlobalNuGetPackages => _configurationDriver.PipelineMode ? SystemGlobalNuGetPackages : _configurationDriver.GlobalNuGetPackages ?? Path.Combine(RunUniqueFolderToSaveGeneratedSolutions, ".nuget"); + public bool IsGlobalPackagesCustomized => GlobalNuGetPackages != SystemGlobalNuGetPackages; } } \ No newline at end of file diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/MSBuildFinder.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/MSBuildFinder.cs index 6631f956d..b7028ebc8 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/MSBuildFinder.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/MSBuildFinder.cs @@ -1,32 +1,74 @@ -using Microsoft.Build.Locator; using System; +using System.IO; using System.Linq; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; namespace Reqnroll.TestProjectGenerator { - public class MSBuildFinder + public class MSBuildFinder(IOutputWriter _outputWriter, ConfigurationDriver _configDriver) { - private static readonly VisualStudioInstanceQueryOptions _visualStudioInstanceQueryOptions = - new() + private string FindUsingVS2022() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return null; + + // Finding paths, like + // C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\MSBuild.exe + + string GetVs2022Path(string edition) { - DiscoveryTypes = - DiscoveryType.DeveloperConsole - | DiscoveryType.VisualStudioSetup - | DiscoveryType.DotNetSdk - }; + var programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles); + return Path.Combine(programFiles, "Microsoft Visual Studio", "2022", edition); + } - public string FindMSBuild() + var editions = new[] { "Enterprise", "Community", "Professional" }; + foreach (var vsPath in editions.Select(GetVs2022Path)) + { + var msBuildPath = Path.Combine(vsPath, "MSBuild", "Current", "Bin", "MSBuild.exe"); + if (File.Exists(msBuildPath)) return msBuildPath; + } + return null; + } + + private string FindUsingVsWhere() { - var instance = MSBuildLocator - .QueryVisualStudioInstances(_visualStudioInstanceQueryOptions) - .FirstOrDefault(); + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return null; - if (instance == null) + //Source: https://github.com/Microsoft/vswhere/wiki/Installing + string GetVsWherePath() { - throw new Exception("Unable to locate MSBuild; Please install the dotnet SDK."); + var programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86); + return Path.Combine(programFiles, "Microsoft Visual Studio", "Installer", "vswhere.exe"); } - return instance.MSBuildPath; + var vsWherePath = GetVsWherePath(); + if (vsWherePath == null || !File.Exists(vsWherePath)) return null; + + // based on instructions at https://github.com/microsoft/vswhere/wiki/Find-MSBuild + const string vsWhereMsBuildParameter = @"-latest -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe"; + var processResult = new ProcessHelper() + .RunProcess(_outputWriter, ".", vsWherePath, vsWhereMsBuildParameter); + + var msBuildPath = + Regex.Split(processResult.CombinedOutput, @"\r?\n") + .FirstOrDefault(l => !string.IsNullOrWhiteSpace(l)); + + if (msBuildPath == null || !File.Exists(msBuildPath)) return null; + + return msBuildPath; + } + + private string FindUsingEnvironmentVariableAndConfig() + { + return _configDriver.MsBuildPath; + } + + public string FindMSBuild() + { + return FindUsingEnvironmentVariableAndConfig() ?? + FindUsingVS2022() ?? + FindUsingVsWhere() ?? + "msbuild"; } } } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/NuGet.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/NuGet.cs index 737786626..99552345b 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/NuGet.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/NuGet.cs @@ -57,7 +57,7 @@ public void Restore(string parameters) protected virtual string GetPathToNuGetExe() { - return Path.Combine(_folders.GlobalPackages, "NuGet.CommandLine".ToLower(), "6.9.1", "tools", "NuGet.exe"); + return Path.Combine(_folders.SystemGlobalNuGetPackages, "NuGet.CommandLine".ToLower(), "6.9.1", "tools", "NuGet.exe"); } } } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProgrammingLanguage.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProgrammingLanguage.cs index 5f7903b50..ff048c99a 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProgrammingLanguage.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProgrammingLanguage.cs @@ -3,9 +3,9 @@ namespace Reqnroll.TestProjectGenerator public enum ProgrammingLanguage { Other, - CSharp, + CSharp73, VB, FSharp, - CSharp10 + CSharp } } \ No newline at end of file diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs index 6d08b61bd..48c1e3ea7 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs @@ -173,8 +173,8 @@ private string GetCode(ProgrammingLanguage language, string csharpcode, string v { switch (language) { + case ProgrammingLanguage.CSharp73: case ProgrammingLanguage.CSharp: - case ProgrammingLanguage.CSharp10: return csharpcode; case ProgrammingLanguage.VB: return vbnetcode; @@ -214,7 +214,7 @@ private void EnsureProjectExists() _project = new Project(ProjectName, ProjectGuid, Language, TargetFramework, Format, ProjectType); - _testProjectFolders.PathToNuGetPackages = _project.ProjectFormat == ProjectFormat.Old ? Path.Combine(_testProjectFolders.PathToSolutionDirectory, "packages") : _folders.RunUniqueGlobalPackages; + _testProjectFolders.PathToNuGetPackages = _project.ProjectFormat == ProjectFormat.Old ? Path.Combine(_testProjectFolders.PathToSolutionDirectory, "packages") : _folders.GlobalNuGetPackages; if (!Directory.Exists(_testProjectFolders.PathToNuGetPackages)) { Directory.CreateDirectory(_testProjectFolders.PathToNuGetPackages); @@ -252,8 +252,8 @@ private void EnsureProjectExists() case ProgrammingLanguage.FSharp: AddInitialFSharpReferences(); break; + case ProgrammingLanguage.CSharp73: case ProgrammingLanguage.CSharp: - case ProgrammingLanguage.CSharp10: AddUnitTestProviderSpecificConfig(); break; } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.csproj b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.csproj index 9b4c2a041..9b6e8fcaf 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.csproj +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.csproj @@ -1,6 +1,6 @@  - net60;net471 + netstandard2.0 full true @@ -16,7 +16,6 @@ - all diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TargetFrameworkMonikerStringBuilder.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TargetFrameworkMonikerStringBuilder.cs index 272c616bb..d0a56b475 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TargetFrameworkMonikerStringBuilder.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TargetFrameworkMonikerStringBuilder.cs @@ -13,6 +13,10 @@ public class TargetFrameworkMonikerStringBuilder [TargetFramework.Net452] = "net452", [TargetFramework.Net461] = "net461", [TargetFramework.Net462] = "net462", + [TargetFramework.Net471] = "net471", + [TargetFramework.Net472] = "net472", + [TargetFramework.Net48] = "net48", + [TargetFramework.Net481] = "net481", [TargetFramework.NetStandard20] = "netstandard2.0", [TargetFramework.Netcoreapp20] = "netcoreapp2.0", [TargetFramework.Netcoreapp21] = "netcoreapp2.1", @@ -20,7 +24,9 @@ public class TargetFrameworkMonikerStringBuilder [TargetFramework.Netcoreapp30] = "netcoreapp3.0", [TargetFramework.Netcoreapp31] = "netcoreapp3.1", [TargetFramework.Net50] = "net5.0", - [TargetFramework.Net60] = "net6.0" + [TargetFramework.Net60] = "net6.0", + [TargetFramework.Net70] = "net7.0", + [TargetFramework.Net80] = "net8.0", }; public string BuildTargetFrameworkMoniker(TargetFramework targetFramework) diff --git a/Reqnroll.sln b/Reqnroll.sln index febc46fd3..8cccaac52 100644 --- a/Reqnroll.sln +++ b/Reqnroll.sln @@ -131,6 +131,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "GitHubWorkflows", "GitHubWo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Reqnroll.BoDi", "Reqnroll.BoDi\Reqnroll.BoDi.csproj", "{8E1AB4D0-98E8-46C5-97EC-7EF47B5DFD27}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Reqnroll.SystemTests", "Tests\Reqnroll.SystemTests\Reqnroll.SystemTests.csproj", "{C658B37D-FD36-4868-9070-4EB452FAE526}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -525,6 +527,18 @@ Global {8E1AB4D0-98E8-46C5-97EC-7EF47B5DFD27}.Release|Any CPU.Build.0 = Release|Any CPU {8E1AB4D0-98E8-46C5-97EC-7EF47B5DFD27}.VS2010IntegrationTest|Any CPU.ActiveCfg = Debug|Any CPU {8E1AB4D0-98E8-46C5-97EC-7EF47B5DFD27}.VS2010IntegrationTest|Any CPU.Build.0 = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Debug-MSTest|Any CPU.ActiveCfg = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Debug-MSTest|Any CPU.Build.0 = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Debug-NUnit|Any CPU.ActiveCfg = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Debug-NUnit|Any CPU.Build.0 = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Debug-XUnit|Any CPU.ActiveCfg = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Debug-XUnit|Any CPU.Build.0 = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.Release|Any CPU.Build.0 = Release|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.VS2010IntegrationTest|Any CPU.ActiveCfg = Debug|Any CPU + {C658B37D-FD36-4868-9070-4EB452FAE526}.VS2010IntegrationTest|Any CPU.Build.0 = Debug|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -565,6 +579,7 @@ Global {2FCD6A28-2134-435E-A205-3D55A6F5A389} = {6070E0CF-FA21-4E82-8A22-3B638CA84525} {C073A609-8A6A-4707-86B0-7BB32EF8ACEE} = {6070E0CF-FA21-4E82-8A22-3B638CA84525} {B6B374C2-ABD8-4F3B-BBF4-505992309168} = {577A0375-1436-446C-802B-3C75C8CEF94F} + {C658B37D-FD36-4868-9070-4EB452FAE526} = {A10B5CD6-38EC-4D7E-9D1C-2EBA8017E437} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A4D0636F-0160-4FA5-81A3-9784C7E3B3A4} diff --git a/Tests/Reqnroll.Specs.Generator.ReqnrollPlugin/TestRunCombinations.cs b/Tests/Reqnroll.Specs.Generator.ReqnrollPlugin/TestRunCombinations.cs index 88fbe8dca..3c3fa35f0 100644 --- a/Tests/Reqnroll.Specs.Generator.ReqnrollPlugin/TestRunCombinations.cs +++ b/Tests/Reqnroll.Specs.Generator.ReqnrollPlugin/TestRunCombinations.cs @@ -13,21 +13,21 @@ public class TestRunCombinations public static List List { get; } = new List() { #if XUNIT_SPECS - new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "Old", TargetFramework = TfmEnumValuenet462, UnitTestProvider = "xUnit", ConfigFormat = "Json"}, - new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "New", TargetFramework = TfmEnumValuenet462, UnitTestProvider = "xUnit", ConfigFormat = "Json"}, - new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "New", TargetFramework = TfmEnumValueNetCore31, UnitTestProvider = "xUnit", ConfigFormat = "Json"}, + new Combination() {ProgrammingLanguage = "CSharp73", ProjectFormat = "Old", TargetFramework = TfmEnumValuenet462, UnitTestProvider = "xUnit", ConfigFormat = "Json"}, + new Combination() {ProgrammingLanguage = "CSharp73", ProjectFormat = "New", TargetFramework = TfmEnumValuenet462, UnitTestProvider = "xUnit", ConfigFormat = "Json"}, + new Combination() {ProgrammingLanguage = "CSharp73", ProjectFormat = "New", TargetFramework = TfmEnumValueNetCore31, UnitTestProvider = "xUnit", ConfigFormat = "Json"}, new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "New", TargetFramework = TfmEnumValueNet60, UnitTestProvider = "xUnit", ConfigFormat = "Json"}, #endif #if MSTEST_SPECS - new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "New", TargetFramework = TfmEnumValuenet462, UnitTestProvider = "MSTest", ConfigFormat = "Json"}, - new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "New", TargetFramework = TfmEnumValueNetCore31, UnitTestProvider = "MSTest", ConfigFormat = "Json"}, + new Combination() {ProgrammingLanguage = "CSharp73", ProjectFormat = "New", TargetFramework = TfmEnumValuenet462, UnitTestProvider = "MSTest", ConfigFormat = "Json"}, + new Combination() {ProgrammingLanguage = "CSharp73", ProjectFormat = "New", TargetFramework = TfmEnumValueNetCore31, UnitTestProvider = "MSTest", ConfigFormat = "Json"}, new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "New", TargetFramework = TfmEnumValueNet60, UnitTestProvider = "MSTest", ConfigFormat = "Json"}, #endif #if NUNIT_SPECS - new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "New", TargetFramework = TfmEnumValuenet462, UnitTestProvider = "NUnit3", ConfigFormat = "Json"}, - new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "New", TargetFramework = TfmEnumValueNetCore31, UnitTestProvider = "NUnit3", ConfigFormat = "Json"}, + new Combination() {ProgrammingLanguage = "CSharp73", ProjectFormat = "New", TargetFramework = TfmEnumValuenet462, UnitTestProvider = "NUnit3", ConfigFormat = "Json"}, + new Combination() {ProgrammingLanguage = "CSharp73", ProjectFormat = "New", TargetFramework = TfmEnumValueNetCore31, UnitTestProvider = "NUnit3", ConfigFormat = "Json"}, new Combination() {ProgrammingLanguage = "CSharp", ProjectFormat = "New", TargetFramework = TfmEnumValueNet60, UnitTestProvider = "NUnit3", ConfigFormat = "Json"}, #endif }; diff --git a/Tests/Reqnroll.Specs/StepDefinitions/ReqnrollConfigurationSteps.cs b/Tests/Reqnroll.Specs/StepDefinitions/ReqnrollConfigurationSteps.cs index f2dc28c86..1ecab808b 100644 --- a/Tests/Reqnroll.Specs/StepDefinitions/ReqnrollConfigurationSteps.cs +++ b/Tests/Reqnroll.Specs/StepDefinitions/ReqnrollConfigurationSteps.cs @@ -1,6 +1,7 @@ using Reqnroll.Specs.Drivers; using Reqnroll.TestProjectGenerator; using Reqnroll.TestProjectGenerator.Driver; +using ConfigurationDriver = Reqnroll.TestProjectGenerator.Driver.ConfigurationDriver; namespace Reqnroll.Specs.StepDefinitions { diff --git a/Tests/Reqnroll.SystemTests/AssemblyAttributes.cs b/Tests/Reqnroll.SystemTests/AssemblyAttributes.cs new file mode 100644 index 000000000..285afd486 --- /dev/null +++ b/Tests/Reqnroll.SystemTests/AssemblyAttributes.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Scope = ExecutionScope.ClassLevel)] diff --git a/Tests/Reqnroll.SystemTests/Drivers/ConsoleOutputConnector.cs b/Tests/Reqnroll.SystemTests/Drivers/ConsoleOutputConnector.cs new file mode 100644 index 000000000..e9978b36d --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Drivers/ConsoleOutputConnector.cs @@ -0,0 +1,31 @@ +using System; +using System.Diagnostics; +using Reqnroll.TestProjectGenerator; + +namespace Reqnroll.SystemTests.Drivers; + +class ConsoleOutputConnector : IOutputWriter +{ + private readonly Stopwatch _stopwatch = new(); + private TimeSpan _previousElapsed = TimeSpan.Zero; + + public ConsoleOutputConnector() + { + _stopwatch.Start(); + WriteLine("Start"); + } + + public void WriteLine(string message) + { + var elapsed = _stopwatch.Elapsed; + message = $"{elapsed:c}(+{(elapsed - _previousElapsed).TotalMilliseconds:0}ms):{message}"; + _previousElapsed = elapsed; + + Console.WriteLine(message); + } + + public void WriteLine(string format, params object[] args) + { + WriteLine(string.Format(format, args)); + } +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Drivers/ExecutionDriver.cs b/Tests/Reqnroll.SystemTests/Drivers/ExecutionDriver.cs new file mode 100644 index 000000000..0668f3a47 --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Drivers/ExecutionDriver.cs @@ -0,0 +1,38 @@ +using Reqnroll.TestProjectGenerator; +using Reqnroll.TestProjectGenerator.Driver; + +namespace Reqnroll.SystemTests.Drivers; + +//ported from Specs +public class ExecutionDriver( + VSTestExecutionDriver _vsTestExecutionDriver, + CompilationDriver _compilationDriver, + TestRunConfiguration _testRunConfiguration) +{ + public void ExecuteTestsWithTag(string tag) + { + _vsTestExecutionDriver.Filter = _testRunConfiguration.UnitTestProvider == UnitTestProvider.xUnit ? + $"Category={tag}" : + $"TestCategory={tag}"; + + ExecuteTests(); + } + + public void ExecuteTests() + { + ExecuteTestsTimes(1); + } + + public void ExecuteTestsTimes(uint times) + { + if (!_compilationDriver.HasTriedToCompile) + { + _compilationDriver.CompileSolution(); + } + + for (uint currentTime = 0; currentTime < times; currentTime++) + { + _vsTestExecutionDriver.ExecuteTests(); + } + } +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Drivers/TestFileManager.cs b/Tests/Reqnroll.SystemTests/Drivers/TestFileManager.cs new file mode 100644 index 000000000..95cb6149b --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Drivers/TestFileManager.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using FluentAssertions; + +namespace Reqnroll.SystemTests.Drivers; + +//ported from Specs +public class TestFileManager +{ + private const string RootNamespace = "Reqnroll.SystemTests"; + private const string TestFileFolder = "Resources"; + private readonly string _prefix = $"{RootNamespace}.{TestFileFolder}"; + + public string GetTestFileContent(string testFileName) + { + var testFileResourceName = testFileName.Replace('/', '.'); + var resourceName = $"{_prefix}.{testFileResourceName}"; + var projectTemplateStream = Assembly + .GetExecutingAssembly() + .GetManifestResourceStream(resourceName); + projectTemplateStream.Should().NotBeNull($"Resource with name '{resourceName}' should be an embedded resource"); + Debug.Assert(projectTemplateStream != null); + string fileContent = new StreamReader(projectTemplateStream).ReadToEnd(); + return fileContent; + } + + public IEnumerable GetTestFeatureFiles() + { + var assembly = Assembly.GetExecutingAssembly(); + string prefixToRemove = $"{_prefix}."; + return assembly.GetManifestResourceNames() + .Where(rn => rn.EndsWith(".feature") && rn.StartsWith(prefixToRemove)) + .Select(rn => rn.Substring(prefixToRemove.Length)); + } +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs new file mode 100644 index 000000000..1afb62a73 --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs @@ -0,0 +1,24 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Reqnroll.SystemTests.Generation; + +[TestCategory("Generation")] +public class GenerationTestBase : SystemTestBase +{ + [TestMethod] + public void GeneratorAllIn_sample_can_be_handled() + { + PrepareGeneratorAllInSamples(); + + ExecuteTests(); + + ShouldAllScenariosPass(); + } + + //TODO: test different outcomes: success, failure, pending, undefined, ignored (scenario & scenario outline) + //TODO: test async steps (async steps are executed in order) + //TODO: test hooks: before/after test run (require special handling by test frameworks) + //TODO: test hooks: before/after test feature & scenario (require special handling by test frameworks) + //TODO: test scenario outlines (nr of examples, params are available in ScenarioContext, allowRowTests=false, examples tags) + //TODO: test parallel execution (details TBD) - maybe this should be in a separate test class +} diff --git a/Tests/Reqnroll.SystemTests/Generation/MsTestGenerationTest.cs b/Tests/Reqnroll.SystemTests/Generation/MsTestGenerationTest.cs new file mode 100644 index 000000000..24bed4a7f --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Generation/MsTestGenerationTest.cs @@ -0,0 +1,47 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator; + +namespace Reqnroll.SystemTests.Generation; + +/// +/// The purpose of these tests to verify that the tests generated by the +/// MsTest generator compile and can execute with MsTest. +/// +[TestClass] +[TestCategory("MsTest")] +public class MsTestGenerationTest : GenerationTestBase +{ + protected override void TestInitialize() + { + base.TestInitialize(); + _testRunConfiguration.UnitTestProvider = UnitTestProvider.MSTest; + } + + /// + /// MsTest v2.* defines the [DataRow] attribute with a ctor that cannot handle if the second + /// parameter of the attribute is a string[]. This causes problems with single-column examples, + /// because in this case the second parameter is a list of example block tags passed in as string[]. + /// + [TestMethod] + public void Generator_handles_tagged_examples_block_with_single_column() + { + AddScenario( + """ + Scenario Outline: Sample Scenario Outline + When happens + @example_tag + Examples: + | what | + | foo | + | bar | + Examples: Second example without tags - in this case the tag list is null. + | what | + | baz | + """); + _projectsDriver.AddPassingStepBinding(); + + ExecuteTests(); + + ShouldAllScenariosPass(3); + } +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Generation/NUnitGenerationTest.cs b/Tests/Reqnroll.SystemTests/Generation/NUnitGenerationTest.cs new file mode 100644 index 000000000..9d4b15b4a --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Generation/NUnitGenerationTest.cs @@ -0,0 +1,19 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator; + +namespace Reqnroll.SystemTests.Generation; + +/// +/// The purpose of these tests to verify that the tests generated by the +/// NUnit generator compile and can execute with NUnit. +/// +[TestClass] +[TestCategory("NUnit")] +public class NUnitGenerationTest : GenerationTestBase +{ + protected override void TestInitialize() + { + base.TestInitialize(); + _testRunConfiguration.UnitTestProvider = UnitTestProvider.NUnit3; + } +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs b/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs new file mode 100644 index 000000000..adf2a569f --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs @@ -0,0 +1,19 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator; + +namespace Reqnroll.SystemTests.Generation; + +/// +/// The purpose of these tests to verify that the tests generated by the +/// xUnit generator compile and can execute with xUnit. +/// +[TestClass] +[TestCategory("xUnit")] +public class XUnitGenerationTest : GenerationTestBase +{ + protected override void TestInitialize() + { + base.TestInitialize(); + _testRunConfiguration.UnitTestProvider = UnitTestProvider.xUnit; + } +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/NuGetPackageVersion.cs b/Tests/Reqnroll.SystemTests/NuGetPackageVersion.cs new file mode 100644 index 000000000..404645feb --- /dev/null +++ b/Tests/Reqnroll.SystemTests/NuGetPackageVersion.cs @@ -0,0 +1,6 @@ +namespace Reqnroll.SystemTests; + +public static class NuGetPackageVersion +{ + public const string Version = "1.0.2-local"; +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/NuGetPackageVersion.template.cs.txt b/Tests/Reqnroll.SystemTests/NuGetPackageVersion.template.cs.txt new file mode 100644 index 000000000..0aeb33570 --- /dev/null +++ b/Tests/Reqnroll.SystemTests/NuGetPackageVersion.template.cs.txt @@ -0,0 +1,6 @@ +namespace Reqnroll.SystemTests; + +public static class NuGetPackageVersion +{ + public const string Version = "NUGET_VERSION"; +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Portability/Net462PortabilityTest.cs b/Tests/Reqnroll.SystemTests/Portability/Net462PortabilityTest.cs new file mode 100644 index 000000000..30db25eba --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Portability/Net462PortabilityTest.cs @@ -0,0 +1,18 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator; +using Reqnroll.TestProjectGenerator.Data; + +namespace Reqnroll.SystemTests.Portability; + +[TestClass] +[TestCategory("NetFramework")] +[TestCategory("Net462")] +public class Net462PortabilityTest : PortabilityTestBase +{ + protected override void TestInitialize() + { + base.TestInitialize(); + _testRunConfiguration.TargetFramework = TargetFramework.Net462; + _testRunConfiguration.ProgrammingLanguage = ProgrammingLanguage.CSharp73; + } +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Portability/Net472PortabilityTest.cs b/Tests/Reqnroll.SystemTests/Portability/Net472PortabilityTest.cs new file mode 100644 index 000000000..d78381efa --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Portability/Net472PortabilityTest.cs @@ -0,0 +1,18 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator; +using Reqnroll.TestProjectGenerator.Data; + +namespace Reqnroll.SystemTests.Portability; + +[TestClass] +[TestCategory("NetFramework")] +[TestCategory("Net472")] +public class Net472PortabilityTest : PortabilityTestBase +{ + protected override void TestInitialize() + { + base.TestInitialize(); + _testRunConfiguration.TargetFramework = TargetFramework.Net472; + _testRunConfiguration.ProgrammingLanguage = ProgrammingLanguage.CSharp73; + } +} diff --git a/Tests/Reqnroll.SystemTests/Portability/Net481PortabilityTest.cs b/Tests/Reqnroll.SystemTests/Portability/Net481PortabilityTest.cs new file mode 100644 index 000000000..2acfb44f6 --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Portability/Net481PortabilityTest.cs @@ -0,0 +1,18 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator; +using Reqnroll.TestProjectGenerator.Data; + +namespace Reqnroll.SystemTests.Portability; + +[TestClass] +[TestCategory("NetFramework")] +[TestCategory("Net481")] +public class Net481PortabilityTest : PortabilityTestBase +{ + protected override void TestInitialize() + { + base.TestInitialize(); + _testRunConfiguration.TargetFramework = TargetFramework.Net481; + _testRunConfiguration.ProgrammingLanguage = ProgrammingLanguage.CSharp73; + } +} diff --git a/Tests/Reqnroll.SystemTests/Portability/Net60PortabilityTest.cs b/Tests/Reqnroll.SystemTests/Portability/Net60PortabilityTest.cs new file mode 100644 index 000000000..79b1c62db --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Portability/Net60PortabilityTest.cs @@ -0,0 +1,16 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator.Data; + +namespace Reqnroll.SystemTests.Portability; + +[TestClass] +[TestCategory("DotNet")] +[TestCategory("Net60")] +public class Net60PortabilityTest : PortabilityTestBase +{ + protected override void TestInitialize() + { + base.TestInitialize(); + _testRunConfiguration.TargetFramework = TargetFramework.Net60; + } +} \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Portability/Net70PortabilityTest.cs b/Tests/Reqnroll.SystemTests/Portability/Net70PortabilityTest.cs new file mode 100644 index 000000000..973d4e845 --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Portability/Net70PortabilityTest.cs @@ -0,0 +1,16 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator.Data; + +namespace Reqnroll.SystemTests.Portability; + +[TestClass] +[TestCategory("DotNet")] +[TestCategory("Net70")] +public class Net70PortabilityTest : PortabilityTestBase +{ + protected override void TestInitialize() + { + base.TestInitialize(); + _testRunConfiguration.TargetFramework = TargetFramework.Net70; + } +} diff --git a/Tests/Reqnroll.SystemTests/Portability/Net80PortabilityTest.cs b/Tests/Reqnroll.SystemTests/Portability/Net80PortabilityTest.cs new file mode 100644 index 000000000..caae4f445 --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Portability/Net80PortabilityTest.cs @@ -0,0 +1,16 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator.Data; + +namespace Reqnroll.SystemTests.Portability; + +[TestClass] +[TestCategory("DotNet")] +[TestCategory("Net80")] +public class Net80PortabilityTest : PortabilityTestBase +{ + protected override void TestInitialize() + { + base.TestInitialize(); + _testRunConfiguration.TargetFramework = TargetFramework.Net80; + } +} diff --git a/Tests/Reqnroll.SystemTests/Portability/PortabilityTestBase.cs b/Tests/Reqnroll.SystemTests/Portability/PortabilityTestBase.cs new file mode 100644 index 000000000..7ff9c6efd --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Portability/PortabilityTestBase.cs @@ -0,0 +1,46 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.TestProjectGenerator.Driver; +using System.Runtime.InteropServices; + +namespace Reqnroll.SystemTests.Portability; + +/// +/// Supported .NET versions: https://docs.reqnroll.net/latest/installation/compatibility.html#net-versions +/// +[TestCategory("Portability")] +public abstract class PortabilityTestBase : SystemTestBase +{ + [TestMethod] + public void GeneratorAllIn_sample_can_be_handled() + { + PrepareGeneratorAllInSamples(); + + ExecuteTests(); + + ShouldAllScenariosPass(); + } + + [TestMethod] + [TestCategory("MsBuild")] + public void GeneratorAllIn_sample_can_be_compiled_with_MsBuild() + { + PrepareGeneratorAllInSamples(); + _compilationDriver.SetBuildTool(BuildTool.MSBuild); + + _compilationDriver.CompileSolution(); + } + + [TestMethod] + [TestCategory("DotnetMSBuild")] + public void GeneratorAllIn_sample_can_be_compiled_with_DotnetMSBuild() + { + PrepareGeneratorAllInSamples(); + _compilationDriver.SetBuildTool(BuildTool.DotnetMSBuild); + + _compilationDriver.CompileSolution(); + } + + //TODO: test different outcomes: success, failure, pending, undefined, ignored (scenario & scenario outline) + //TODO: test async steps (async steps are executed in order) + //TODO: test before/after test run hooks (.NET Framework version of Reqnroll is subscribed to assembly unload) +} diff --git a/Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj b/Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj new file mode 100644 index 000000000..4ef30b6fe --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj @@ -0,0 +1,51 @@ + + + + net8.0 + enable + false + $(NoWarn);CS2002 + + + + + + + + + + + + + + + + + + + + + false + + + false + + + false + + + false + + + + + + + + + + + + + + diff --git a/Tests/Reqnroll.SystemTests/Resources/GeneratorAllInSample1.feature b/Tests/Reqnroll.SystemTests/Resources/GeneratorAllInSample1.feature new file mode 100644 index 000000000..97c1334ec --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Resources/GeneratorAllInSample1.feature @@ -0,0 +1,81 @@ +Feature: Generator "all in" Sample 1 + +Sample feature file that contains various different Gherkin elements to test +generation. + +In this file we demonstrate +* Scenarios +* Scenario Steps +* Scenario Step arguments: DataTable, DocString +* Scenario tags +* Feature & scenario descriptions +* Comments + +See also GeneratorSample2.feature + +# this is a comment +@tag1 @tag2 +Scenario: Basic scenario with steps + This text describes details about the purpose + of the scenario. + + Given there is something + And there is something else + When something happens + Then something should have happened + +@single_tag +Scenario: Scenario with DataTable + When something happens with + | who | when | + | me | today | + | someone else | tomorrow | + +Scenario: Scenario with DocString + When something happens as + """ + Something happens to + - me + - and someone else + """ + +Scenario Outline: Scenario outline with different placeholders + Given there is + And there is something else + When happens with + | who | when | + | me | | + | someone else | tomorrow | + When something should have happened as + """ + Something happens to + - me + - and someone else tomorrow + """ +Examples: + | what | when | not used | + | something | today | this is not used | + | something else | tomorrow | this is not used | + | special characters \| " ' | today | should work still | + +@tag1 +Scenario Outline: Scenario outline with multiple examples blocks + Given there is + And there is something else + When happens + When something should have happened + +Examples: + | what | when | not used | + | something | today | this is not used | + | something else | tomorrow | this is not used | +@tag2 +Examples: + | what | when | not used | + | foo | bar | baz | + | qux | quux | corge | +@tag3 @tag4 +Examples: Example block name + | what | when | not used | + | grault | garply | waldo | + | fred | plugh | xyzzy | diff --git a/Tests/Reqnroll.SystemTests/Resources/GeneratorAllInSample2.feature b/Tests/Reqnroll.SystemTests/Resources/GeneratorAllInSample2.feature new file mode 100644 index 000000000..1d946b84f --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Resources/GeneratorAllInSample2.feature @@ -0,0 +1,40 @@ +Feature: Generator "all in" Sample 2 + +Sample feature file that contains various different Gherkin elements to test +generation. + +In this file we demonstrate +* Backgrounds +* Rules +* Rule Backgrounds + +See also GeneratorSample1.feature + +Background: + Given there is something + +Scenario: Scenario outside of rules + When something happens + +@rule_tag1 +Rule: Rule with single scenario +This description belongs to the rule + +Scenario: Single scenario in a rule + When something happens + +Rule: Rule with multiple scenarios + +Scenario: First scenario in a rule + When something happens + +Scenario: Second scenario in a rule + When something happens + +Rule: Rule with background + +Background: Rule background + Given there is something else + +Scenario: Scenario in a rule with background + When something happens diff --git a/Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs b/Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs new file mode 100644 index 000000000..d64f4d6ab --- /dev/null +++ b/Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs @@ -0,0 +1,23 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Reqnroll.SystemTests.Smoke; + +[TestClass] +[TestCategory("Smoke")] +public class SmokeTest : SystemTestBase +{ + [TestMethod] + public void Handles_the_simplest_scenario() + { + AddScenario( + """ + Scenario: Sample Scenario + When something happens + """); + _projectsDriver.AddPassingStepBinding(); + + ExecuteTests(); + + ShouldAllScenariosPass(); + } +} diff --git a/Tests/Reqnroll.SystemTests/SystemTestBase.cs b/Tests/Reqnroll.SystemTests/SystemTestBase.cs new file mode 100644 index 000000000..61dd12c7b --- /dev/null +++ b/Tests/Reqnroll.SystemTests/SystemTestBase.cs @@ -0,0 +1,149 @@ +using System; +using System.Text.RegularExpressions; +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Reqnroll.BoDi; +using Reqnroll.SystemTests.Drivers; +using Reqnroll.TestProjectGenerator; +using Reqnroll.TestProjectGenerator.Data; +using Reqnroll.TestProjectGenerator.Driver; +using Reqnroll.TestProjectGenerator.Helpers; + +namespace Reqnroll.SystemTests; +public abstract class SystemTestBase +{ + protected ProjectsDriver _projectsDriver = null!; + protected ExecutionDriver _executionDriver = null!; + protected VSTestExecutionDriver _vsTestExecutionDriver = null!; + protected TestFileManager _testFileManager = new(); + protected FolderCleaner _folderCleaner = null!; + protected ObjectContainer _testContainer = null!; + protected TestRunConfiguration _testRunConfiguration = null!; + protected CurrentVersionDriver _currentVersionDriver = null!; + protected CompilationDriver _compilationDriver = null!; + + protected int _preparedTests = 0; + + public TestContext TestContext { get; set; } = null!; + + [TestInitialize] + public void TestInitializeMethod() + { + TestInitialize(); + } + + protected virtual void TestInitialize() + { + _testContainer = new ObjectContainer(); + _testContainer.RegisterTypeAs(); + + _testRunConfiguration = _testContainer.Resolve(); + _testRunConfiguration.ProgrammingLanguage = ProgrammingLanguage.CSharp; + _testRunConfiguration.ProjectFormat = ProjectFormat.New; + _testRunConfiguration.ConfigurationFormat = ConfigurationFormat.Json; + _testRunConfiguration.TargetFramework = TargetFramework.Net80; + _testRunConfiguration.UnitTestProvider = UnitTestProvider.MSTest; + + _currentVersionDriver = _testContainer.Resolve(); + _currentVersionDriver.NuGetVersion = NuGetPackageVersion.Version; + _currentVersionDriver.ReqnrollNuGetVersion = NuGetPackageVersion.Version; + + _folderCleaner = _testContainer.Resolve(); + _folderCleaner.EnsureOldRunFoldersCleaned(); + + _projectsDriver = _testContainer.Resolve(); + _executionDriver = _testContainer.Resolve(); + _vsTestExecutionDriver = _testContainer.Resolve(); + _compilationDriver = _testContainer.Resolve(); + } + + protected void AddFeatureFileFromResource(string fileName, int? preparedTests = null) + { + var featureFileContent = _testFileManager.GetTestFileContent(fileName); + AddFeatureFile(featureFileContent, preparedTests); + } + + private int? GetTestCount(string gherkinContent) + { + var matches = Regex.Matches(gherkinContent, @"^\s*((?Scenario:|Scenario Outline:)|(?Examples:)|(?\|)).*?\s*$", RegexOptions.Multiline); + int count = 0; + var inExamples = false; + foreach (Match match in matches) + { + if (match.Groups["scenario"].Success) + { + count++; + inExamples = false; + } + else if (match.Groups["exmaples"].Success) + { + if (!inExamples) + count--; // was a scenario outline + count--; // examples header row + inExamples = true; + } + else if (inExamples && match.Groups["row"].Success) + { + count++; + } + } + + if (count == 0) + { + Console.WriteLine("No tests detected"); + return null; + } + + Console.WriteLine($"Detected {count} tests"); + return count; + } + + protected void AddFeatureFile(string featureFileContent, int? preparedTests = null) + { + _projectsDriver.AddFeatureFile(featureFileContent); + UpdatePreparedTests(featureFileContent, preparedTests); + } + + protected void AddScenario(string scenarioContent, int? preparedTests = null) + { + _projectsDriver.AddScenario(scenarioContent); + UpdatePreparedTests(scenarioContent, preparedTests); + } + + private void UpdatePreparedTests(string gherkinContent, int? preparedTests) + { + if (preparedTests == null) + { + var match = Regex.Match(gherkinContent, @"^#TEST_COUNT:\s*(?\d+)\s*$", RegexOptions.Multiline); + preparedTests = match.Success ? + int.Parse(match.Groups["count"].Value) : + GetTestCount(gherkinContent); + } + + if (preparedTests != null) _preparedTests += preparedTests.Value; + } + + protected void PrepareGeneratorAllInSamples() + { + AddFeatureFileFromResource("GeneratorAllInSample1.feature"); + AddFeatureFileFromResource("GeneratorAllInSample2.feature"); + _projectsDriver.AddPassingStepBinding(); + } + + protected void ExecuteTests() + { + _executionDriver.ExecuteTests(); + } + + protected void ShouldAllScenariosPass(int? expectedNrOfTestsSpec = null) + { + if (expectedNrOfTestsSpec == null && _preparedTests == 0) + throw new ArgumentException($"If {nameof(_preparedTests)} is not set, the {nameof(expectedNrOfTestsSpec)} is mandatory.", nameof(expectedNrOfTestsSpec)); + var expectedNrOfTests = expectedNrOfTestsSpec ?? _preparedTests; + + _vsTestExecutionDriver.LastTestExecutionResult.Should().NotBeNull(); + _vsTestExecutionDriver.LastTestExecutionResult.Total.Should().Be(expectedNrOfTests, $"the run should contain {expectedNrOfTests} tests"); + _vsTestExecutionDriver.LastTestExecutionResult.Succeeded.Should().Be(expectedNrOfTests, "all tests should pass"); + _folderCleaner.CleanSolutionFolder(); + } +}