Introduce System tests (#82)

* system tests concept

# Conflicts:
#	Reqnroll.sln

* Update test for problem identified for #54 by @livioc

* replace BoDi with Reqnroll.BoDi

* portability test POC

* Undo 'Microsoft.Build.Locator' for MsBuild location and make it flexible with env vars (#75)

* add portability compilation tests with msbuild

* Add system tests to CI

* Fix system tests to CI

* Add TODOs for further portability tests

* Fix system tests to CI, take 2

* Change SystemTests to use MsTest

* set default target framework to .NET 8.0

* Add .NET Information to CI

* test specs filter on CI

* Revert "test specs filter on CI"

This reverts commit bdcdbaa.

* fix specs_filter on CI

* speed up build by using heuristics to find MsBuild

* speed up build by using global nuget packages folder

* cleanup CI script

* add CI system test for linux

* create Generation test structure

* add more categories and exclude .NET Framework and MsBuild tests from linux

* Add NUnit and xUnit tests

* rename AppConfigDriver to ConfigurationDriver, allow customizing nuget global folder

* fix strange test error if .NET 4.6.2 test target was not included in run

* Add smoke test

* TEST: remove test filter from Linux CI

* Add Generation test TODO comments

* re-add filter to linux CI

* enable parallel system testing

* set target fw of Reqnroll.TestProjectGenerator to netstandard2.0

* re-enable all specs tests

* code cleanup

.github/workflows/ci.yml

@@ -37,6 +37,7 @@ permissions:
 
 env:
   SPECS_FILTER: "" # use for testing CI: "&Category=basicExecution"
+  REQNROLL_TEST_PIPELINEMODE: true
 
 jobs:
   build:
@@ -48,6 +49,7 @@ jobs:
       product_configuration: ${{ steps.versions.outputs.product_configuration }}
       build_params: ${{ steps.versions.outputs.build_params }}
       test_params: ${{ steps.versions.outputs.test_params }}
+      specs_filter: ${{ steps.versions.outputs.specs_filter }}
 
     steps:
     - uses: actions/checkout@v4
@@ -97,7 +99,7 @@ jobs:
             $specsFilter = "&$specsFilter"
         }
         else {
-            $specsFilter = $envSPECS_FILTER
+            $specsFilter = $env:SPECS_FILTER
         }
         Write-Output "specs_filter=$specsFilter" >> $env:GITHUB_OUTPUT
         Write-Output "Specs Filter: $specsFilter"
@@ -248,3 +250,67 @@ jobs:
       with:
         name: specs-mstest-trx
         path: "**/specs-mstest-results.trx"
+
+  system-tests-windows:
+    runs-on: windows-latest
+    needs: build
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 # avoid shallow clone so nbgv can do its work.
+    - name: .NET Information
+      run: |
+        dotnet --list-sdks
+        dotnet --list-runtimes
+    - name: Restore dependencies
+      run: dotnet restore
+    - name: Install Test Report Dependencies
+      run: | 
+        dotnet add ./Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj package GitHubActionsTestLogger
+    - name: Build
+      run: dotnet build --no-restore ${{ needs.build.outputs.build_params }}
+    - name: System Tests
+      shell: pwsh
+      run: |
+        $gitHubActionsLoggerSettings = '"GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true;annotations.titleFormat=[@traits.Category] @test;annotations.messageFormat=@error\n@trace"'
+        dotnet test ./Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj ${{ needs.build.outputs.test_params }} --logger "trx;LogFileName=systemtests-windows-results.trx" --logger $gitHubActionsLoggerSettings -- RunConfiguration.CollectSourceInformation=true
+    - name: Upload Test Result TRX Files
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: systemtests-windows-trx
+        path: "**/*.trx"
+
+
+  system-tests-linux:
+    runs-on: ubuntu-latest
+    needs: build
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 # avoid shallow clone so nbgv can do its work.
+    - name: .NET Information
+      run: |
+        dotnet --list-sdks
+        dotnet --list-runtimes
+    - name: Restore dependencies
+      run: dotnet restore
+    - name: Install Test Report Dependencies
+      run: | 
+        dotnet add ./Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj package GitHubActionsTestLogger
+    - name: Build
+      run: dotnet build --no-restore ${{ needs.build.outputs.build_params }}
+    - name: System Tests
+      shell: pwsh
+      run: |
+        $gitHubActionsLoggerSettings = '"GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true;annotations.titleFormat=[@traits.Category] @test;annotations.messageFormat=@error\n@trace"'
+        dotnet test ./Tests/Reqnroll.SystemTests/Reqnroll.SystemTests.csproj ${{ needs.build.outputs.test_params }} --filter "TestCategory!=MsBuild&TestCategory!=Net481" --logger "trx;LogFileName=systemtests-linux-results.trx" --logger $gitHubActionsLoggerSettings -- RunConfiguration.CollectSourceInformation=true
+    - name: Upload Test Result TRX Files
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: systemtests-linux-trx
+        path: "**/*.trx"
+

.ncrunch/Reqnroll.TestProjectGenerator.v3.ncrunchproject

@@ -1,7 +1,3 @@
 <ProjectConfiguration>
-  <Settings>
-    <CustomBuildProperties>
-      <Value>TargetFrameworks = net60</Value>
-    </CustomBuildProperties>
-  </Settings>
+  <Settings />
 </ProjectConfiguration>

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/NuGetConfigGeneratorTests.cs

@@ -1,6 +1,5 @@
 using FluentAssertions;
 using Reqnroll.TestProjectGenerator.Data;
-using Reqnroll.TestProjectGenerator.NewApi._1_Memory;
 using Xunit;
 
 namespace Reqnroll.TestProjectGenerator.Tests

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/ProjectTests.cs

@@ -238,20 +238,19 @@ private string GetProjectFolderPath(string solutionFolder, Project project)
         }
 
         [Fact]
-        public void CreateEmtpyCSharpProjectInNewFormat()
+        public void CreateEmptyCSharpProjectInNewFormat()
         {
             var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp);
 
             new SolutionWriter(new Mock<IOutputWriter>().Object).WriteToFileSystem(solution, solutionFolder);
 
             string projectFileContent = GetProjectFileContent(solutionFolder, project);
 
-            projectFileContent.Should()
-                              .Contain("<Project Sdk=\"Microsoft.NET.Sdk\">\r\n  <PropertyGroup>\r\n    <TargetFramework>net462</TargetFramework>\r\n  </PropertyGroup>\r\n</Project>");
+            projectFileContent.Should().Contain("<Project Sdk=\"Microsoft.NET.Sdk\">");
         }
 
         [Fact]
-        public void CreateEmtpyCSharpCore3_1ProjectInNewFormat()
+        public void CreateEmptyCSharpCore3_1ProjectInNewFormat()
         {
             var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Netcoreapp31);
 
@@ -264,20 +263,20 @@ public void CreateEmtpyCSharpCore3_1ProjectInNewFormat()
         }
 
         [Fact]
-        public void CreateEmtpyCSharpNet50ProjectInNewFormat()
+        public void CreateEmptyCSharpNet50ProjectInNewFormat()
         {
-            var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Net50);
+            var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp73, TargetFramework.Net50);
 
             new SolutionWriter(new Mock<IOutputWriter>().Object).WriteToFileSystem(solution, solutionFolder);
 
             string projectFileContent = GetProjectFileContent(solutionFolder, project);
 
             projectFileContent.Should()
-                .Contain("<Project Sdk=\"Microsoft.NET.Sdk\">\r\n  <PropertyGroup>\r\n    <TargetFramework>net5.0</TargetFramework>\r\n  </PropertyGroup>\r\n</Project>");
+                .Contain("<Project Sdk=\"Microsoft.NET.Sdk\">\r\n  <PropertyGroup>\r\n    <TargetFramework>net5.0</TargetFramework>\r\n    <LangVersion>7.3</LangVersion>\r\n  </PropertyGroup>\r\n</Project>");
         }
 
         [Fact]
-        public void CreateEmtpyCSharpNet60ProjectInNewFormat()
+        public void CreateEmptyCSharpNet60ProjectInNewFormat()
         {
             var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Net60);
 
@@ -290,7 +289,72 @@ public void CreateEmtpyCSharpNet60ProjectInNewFormat()
         }
 
         [Fact]
-        public void CreateEmtpyFSharpProjectInNewFormat()
+        public void CreateEmptyCSharpNet70ProjectInNewFormat()
+        {
+            var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Net70);
+
+            new SolutionWriter(new Mock<IOutputWriter>().Object).WriteToFileSystem(solution, solutionFolder);
+
+            string projectFileContent = GetProjectFileContent(solutionFolder, project);
+
+            projectFileContent.Should()
+                              .Contain("<Project Sdk=\"Microsoft.NET.Sdk\">\r\n  <PropertyGroup>\r\n    <TargetFramework>net7.0</TargetFramework>\r\n    <ImplicitUsings>enable</ImplicitUsings>\r\n    <Nullable>enable</Nullable>\r\n  </PropertyGroup>\r\n</Project>");
+        }
+
+        [Fact]
+        public void CreateEmptyCSharpNet80ProjectInNewFormat()
+        {
+            var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp, TargetFramework.Net80);
+
+            new SolutionWriter(new Mock<IOutputWriter>().Object).WriteToFileSystem(solution, solutionFolder);
+
+            string projectFileContent = GetProjectFileContent(solutionFolder, project);
+
+            projectFileContent.Should()
+                              .Contain("<Project Sdk=\"Microsoft.NET.Sdk\">\r\n  <PropertyGroup>\r\n    <TargetFramework>net8.0</TargetFramework>\r\n    <ImplicitUsings>enable</ImplicitUsings>\r\n    <Nullable>enable</Nullable>\r\n  </PropertyGroup>\r\n</Project>");
+        }
+
+        [Fact]
+        public void CreateEmptyCSharpNet481ProjectInNewFormat()
+        {
+            var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp73, TargetFramework.Net481);
+
+            new SolutionWriter(new Mock<IOutputWriter>().Object).WriteToFileSystem(solution, solutionFolder);
+
+            string projectFileContent = GetProjectFileContent(solutionFolder, project);
+
+            projectFileContent.Should()
+                              .Contain("<Project Sdk=\"Microsoft.NET.Sdk\">\r\n  <PropertyGroup>\r\n    <TargetFramework>net481</TargetFramework>\r\n    <LangVersion>7.3</LangVersion>\r\n  </PropertyGroup>\r\n</Project>");
+        }
+
+        [Fact]
+        public void CreateEmptyCSharpNet462ProjectInNewFormat()
+        {
+            var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp73, TargetFramework.Net462);
+
+            new SolutionWriter(new Mock<IOutputWriter>().Object).WriteToFileSystem(solution, solutionFolder);
+
+            string projectFileContent = GetProjectFileContent(solutionFolder, project);
+
+            projectFileContent.Should()
+                              .Contain("<Project Sdk=\"Microsoft.NET.Sdk\">\r\n  <PropertyGroup>\r\n    <TargetFramework>net462</TargetFramework>\r\n    <LangVersion>7.3</LangVersion>\r\n  </PropertyGroup>\r\n</Project>");
+        }
+
+        [Fact]
+        public void CreateEmptyCSharpNet472ProjectInNewFormat()
+        {
+            var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.CSharp73, TargetFramework.Net472);
+
+            new SolutionWriter(new Mock<IOutputWriter>().Object).WriteToFileSystem(solution, solutionFolder);
+
+            string projectFileContent = GetProjectFileContent(solutionFolder, project);
+
+            projectFileContent.Should()
+                              .Contain("<Project Sdk=\"Microsoft.NET.Sdk\">\r\n  <PropertyGroup>\r\n    <TargetFramework>net472</TargetFramework>\r\n    <LangVersion>7.3</LangVersion>\r\n  </PropertyGroup>\r\n</Project>");
+        }
+
+        [Fact]
+        public void CreateEmptyFSharpProjectInNewFormat()
         {
             var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.FSharp);
 
@@ -303,7 +367,7 @@ public void CreateEmtpyFSharpProjectInNewFormat()
         }
 
         [Fact]
-        public void CreateEmtpyVbProjectInNewFormat()
+        public void CreateEmptyVbProjectInNewFormat()
         {
             var (solution, project, solutionFolder) = CreateEmptySolutionAndProject(ProjectFormat.New, ProgrammingLanguage.VB);
 

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Compiler.cs

@@ -34,15 +34,7 @@ public CompileResult Run(BuildTool buildTool, bool? treatWarningsAsErrors)
 
         private CompileResult CompileWithMSBuild(bool? treatWarningsAsErrors)
         {
-            string msBuildPath="";
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-            {
-                msBuildPath = _msBuildFinder.FindMSBuild();
-            }
-            else
-            {
-                msBuildPath = "msbuild";
-            }
+            var msBuildPath = _msBuildFinder.FindMSBuild();
 
             _outputWriter.WriteLine($"Invoke MsBuild from {msBuildPath}");
 

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ConfigurationDriver.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Configuration;
+
+namespace Reqnroll.TestProjectGenerator;
+
+public class ConfigurationDriver
+{
+    public string TestProjectFolderName => GetConfigSetting("testProjectFolder", "RR");
+    public string VSTestPath => GetConfigSetting("vstestPath", "Common7\\IDE\\CommonExtensions\\Microsoft\\TestWindow");
+    public string MsBuildPath => GetConfigSetting(nameof(MsBuildPath));
+    public bool PipelineMode => GetConfigSwitch(nameof(PipelineMode));
+    public string GlobalNuGetPackages => GetConfigSetting(nameof(GlobalNuGetPackages));
+
+    public bool GetConfigSwitch(string key, bool defaultValue = false)
+    {
+        return GetConfigSetting(key, defaultValue.ToString()).Equals("true", StringComparison.InvariantCultureIgnoreCase);
+    }
+
+    public string GetConfigSetting(string key, string defaultValue = null)
+    {
+        var envSetting = Environment.GetEnvironmentVariable($"REQNROLL_TEST_{key}".ToUpperInvariant());
+        if (!string.IsNullOrEmpty(envSetting)) return envSetting;
+
+        var configSetting = ConfigurationManager.AppSettings[key];
+        if (!string.IsNullOrEmpty(configSetting)) return configSetting;
+
+        return defaultValue;
+    }
+}

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/TargetFramework.cs

@@ -13,7 +13,13 @@ public enum TargetFramework
         Netcoreapp31,
         Net461,
         Net462,
+        Net471,
+        Net472,
+        Net48,
+        Net481,
         Net50,
-        Net60
+        Net60,
+        Net70,
+        Net80
     }
 }

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Dotnet/NewCommandBuilder.NewProjectCommandBuilder.cs

@@ -51,11 +51,16 @@ protected override string BuildArguments()
                 arguments = AddArgument(
                     arguments,
                     "-lang",
+                    _language == ProgrammingLanguage.CSharp73 ? "\"C#\"" :
                     _language == ProgrammingLanguage.CSharp ? "\"C#\"" :
-                    _language == ProgrammingLanguage.CSharp10 ? "\"C#\"" :
                     _language == ProgrammingLanguage.VB ? "VB" :
                     _language == ProgrammingLanguage.FSharp ? "\"F#\"" : string.Empty);
 
+                if (_language == ProgrammingLanguage.CSharp73)
+                {
+                    arguments = AddArgument(arguments, "--langVersion", "7.3");
+                }
+
                 return arguments;
             }
         }

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/SolutionDriver.cs

@@ -17,6 +17,7 @@ public class SolutionDriver
         private readonly ProjectBuilderFactory _projectBuilderFactory;
         private readonly Folders _folders;
         private readonly ArtifactNamingConvention _artifactNamingConvention;
+        private readonly IOutputWriter _outputWriter;
         private readonly Solution _solution;
         private readonly Dictionary<string, ProjectBuilder> _projects = new Dictionary<string, ProjectBuilder>();
         private ProjectBuilder _defaultProject;
@@ -27,18 +28,20 @@ public SolutionDriver(
             ProjectBuilderFactory projectBuilderFactory,
             Folders folders,
             TestProjectFolders testProjectFolders,
-            ArtifactNamingConvention artifactNamingConvention)
+            ArtifactNamingConvention artifactNamingConvention,
+            IOutputWriter outputWriter)
         {
             _nuGetConfigGenerator = nuGetConfigGenerator;
             _testRunConfiguration = testRunConfiguration;
             _projectBuilderFactory = projectBuilderFactory;
             _folders = folders;
             _artifactNamingConvention = artifactNamingConvention;
+            _outputWriter = outputWriter;
             NuGetSources = new List<NuGetSource>
             {
-                new NuGetSource("LocalReqnrollDevPackages", _folders.NuGetFolder),
-                new NuGetSource("Reqnroll CI", "https://www.myget.org/F/reqnroll/api/v3/index.json"),
-                new NuGetSource("Reqnroll Unstable", "https://www.myget.org/F/reqnroll-unstable/api/v3/index.json")
+                new("LocalReqnrollDevPackages", _folders.NuGetFolder),
+                new("Reqnroll CI", "https://www.myget.org/F/reqnroll/api/v3/index.json"),
+                new("Reqnroll Unstable", "https://www.myget.org/F/reqnroll-unstable/api/v3/index.json")
             };
 
             if (testRunConfiguration.UnitTestProvider == UnitTestProvider.SpecRun)
@@ -86,7 +89,10 @@ public Solution GetSolution()
                 _solution.AddProject(project);
             }
 
-            _solution.NugetConfig = _nuGetConfigGenerator?.Generate(NuGetSources.ToArray(), _folders.RunUniqueGlobalPackages);
+            var customGlobalPackagesFolder = _folders.IsGlobalPackagesCustomized ? _folders.GlobalNuGetPackages : null;
+            if (customGlobalPackagesFolder != null)
+                _outputWriter.WriteLine($"Using custom global packages folder: {customGlobalPackagesFolder}");
+            _solution.NugetConfig = _nuGetConfigGenerator?.Generate(NuGetSources.ToArray(), customGlobalPackagesFolder);
             return _solution;
         }