Get IronPigeon building and testing on Linux (#31)

* Get IronPigeon building and testing on Linux

* TEST revert

* TEMP revert

* TEMP put emulators back

* TEMP reapply

* TEMP re-remove Platform switch from test commands

* TEMP bring back env

* Remove env var redirection

* Skip failing test on Linux

* Remove cyclical variable

* Supply connection string just to linux/mac

* Bring back platform switch

* Remove --no-build

* finished?

IronPigeon.sln

@@ -33,37 +33,63 @@ EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
+		Debug|Non_Windows = Debug|Non_Windows
 		Release|Any CPU = Release|Any CPU
+		Release|Non_Windows = Release|Non_Windows
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{BEFB82D5-C686-4D50-91C7-419846C8F201}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{BEFB82D5-C686-4D50-91C7-419846C8F201}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BEFB82D5-C686-4D50-91C7-419846C8F201}.Debug|Non_Windows.ActiveCfg = Debug|Any CPU
+		{BEFB82D5-C686-4D50-91C7-419846C8F201}.Debug|Non_Windows.Build.0 = Debug|Any CPU
 		{BEFB82D5-C686-4D50-91C7-419846C8F201}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{BEFB82D5-C686-4D50-91C7-419846C8F201}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BEFB82D5-C686-4D50-91C7-419846C8F201}.Release|Non_Windows.ActiveCfg = Release|Any CPU
+		{BEFB82D5-C686-4D50-91C7-419846C8F201}.Release|Non_Windows.Build.0 = Release|Any CPU
 		{98AFB63C-821F-491F-B8CF-394201693F2A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{98AFB63C-821F-491F-B8CF-394201693F2A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{98AFB63C-821F-491F-B8CF-394201693F2A}.Debug|Non_Windows.ActiveCfg = Debug|Any CPU
 		{98AFB63C-821F-491F-B8CF-394201693F2A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{98AFB63C-821F-491F-B8CF-394201693F2A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{98AFB63C-821F-491F-B8CF-394201693F2A}.Release|Non_Windows.ActiveCfg = Release|Any CPU
 		{7413FF6F-DC39-4B86-AD14-EAC145F5C3B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{7413FF6F-DC39-4B86-AD14-EAC145F5C3B7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7413FF6F-DC39-4B86-AD14-EAC145F5C3B7}.Debug|Non_Windows.ActiveCfg = Debug|Any CPU
+		{7413FF6F-DC39-4B86-AD14-EAC145F5C3B7}.Debug|Non_Windows.Build.0 = Debug|Any CPU
 		{7413FF6F-DC39-4B86-AD14-EAC145F5C3B7}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7413FF6F-DC39-4B86-AD14-EAC145F5C3B7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7413FF6F-DC39-4B86-AD14-EAC145F5C3B7}.Release|Non_Windows.ActiveCfg = Release|Any CPU
+		{7413FF6F-DC39-4B86-AD14-EAC145F5C3B7}.Release|Non_Windows.Build.0 = Release|Any CPU
 		{4DE86407-877D-4C6B-B457-1236B20F8823}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{4DE86407-877D-4C6B-B457-1236B20F8823}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4DE86407-877D-4C6B-B457-1236B20F8823}.Debug|Non_Windows.ActiveCfg = Debug|Any CPU
 		{4DE86407-877D-4C6B-B457-1236B20F8823}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{4DE86407-877D-4C6B-B457-1236B20F8823}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4DE86407-877D-4C6B-B457-1236B20F8823}.Release|Non_Windows.ActiveCfg = Release|Any CPU
 		{6995C553-7AE1-490E-BCF2-406E105EDEF8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{6995C553-7AE1-490E-BCF2-406E105EDEF8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6995C553-7AE1-490E-BCF2-406E105EDEF8}.Debug|Non_Windows.ActiveCfg = Debug|Any CPU
+		{6995C553-7AE1-490E-BCF2-406E105EDEF8}.Debug|Non_Windows.Build.0 = Debug|Any CPU
 		{6995C553-7AE1-490E-BCF2-406E105EDEF8}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{6995C553-7AE1-490E-BCF2-406E105EDEF8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6995C553-7AE1-490E-BCF2-406E105EDEF8}.Release|Non_Windows.ActiveCfg = Release|Any CPU
+		{6995C553-7AE1-490E-BCF2-406E105EDEF8}.Release|Non_Windows.Build.0 = Release|Any CPU
 		{ABD9D7E0-665C-44D8-930D-14B5785F832F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{ABD9D7E0-665C-44D8-930D-14B5785F832F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ABD9D7E0-665C-44D8-930D-14B5785F832F}.Debug|Non_Windows.ActiveCfg = Debug|Any CPU
+		{ABD9D7E0-665C-44D8-930D-14B5785F832F}.Debug|Non_Windows.Build.0 = Debug|Any CPU
 		{ABD9D7E0-665C-44D8-930D-14B5785F832F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{ABD9D7E0-665C-44D8-930D-14B5785F832F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ABD9D7E0-665C-44D8-930D-14B5785F832F}.Release|Non_Windows.ActiveCfg = Release|Any CPU
+		{ABD9D7E0-665C-44D8-930D-14B5785F832F}.Release|Non_Windows.Build.0 = Release|Any CPU
 		{B9CDFDA2-C9A9-425D-B53B-E6E188CC996C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{B9CDFDA2-C9A9-425D-B53B-E6E188CC996C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B9CDFDA2-C9A9-425D-B53B-E6E188CC996C}.Debug|Non_Windows.ActiveCfg = Debug|Any CPU
+		{B9CDFDA2-C9A9-425D-B53B-E6E188CC996C}.Debug|Non_Windows.Build.0 = Debug|Any CPU
 		{B9CDFDA2-C9A9-425D-B53B-E6E188CC996C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B9CDFDA2-C9A9-425D-B53B-E6E188CC996C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B9CDFDA2-C9A9-425D-B53B-E6E188CC996C}.Release|Non_Windows.ActiveCfg = Release|Any CPU
+		{B9CDFDA2-C9A9-425D-B53B-E6E188CC996C}.Release|Non_Windows.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

azure-pipelines/build.yml

@@ -4,6 +4,8 @@ parameters:
 jobs:
 - job: Windows
   pool: ${{ parameters.windowsPool }}
+  variables:
+    BuildPlatform: Any CPU
   steps:
   - checkout: self
     clean: true
@@ -12,32 +14,48 @@ jobs:
   - powershell: '& (./azure-pipelines/Get-nbgv.ps1) cloud'
     displayName: Set build number
 
+  - pwsh: ./tools/Emulators.ps1
+    displayName: Start storage emulators
+
   - template: dotnet.yml
 
 - job: Linux
-  condition: false
   pool:
     vmImage: Ubuntu 18.04
+  variables:
+    BuildPlatform: Non_Windows
   steps:
   - checkout: self
     clean: true
   - template: install-dependencies.yml
+  - bash: ./tools/Emulators.sh &
+    displayName: Start storage emulators
   - template: dotnet.yml
+    parameters:
+      test_secrets:
+        CosmosDBConnectionString: $(CosmosDBConnectionString)
 
 - job: macOS
   condition: false
   pool:
     vmImage: macOS-10.15
+  variables:
+    BuildPlatform: Non_Windows
   steps:
   - checkout: self
     clean: true
   - template: install-dependencies.yml
+  - bash: ./tools/Emulators.sh &
+    displayName: Start storage emulators
   - template: dotnet.yml
+    parameters:
+      test_secrets:
+        CosmosDBConnectionString: $(CosmosDBConnectionString)
 
 - job: WrapUp
   dependsOn:
   - Windows
-#  - Linux
+  - Linux
 #  - macOS
   pool: ${{ parameters.windowsPool }}
   condition: succeededOrFailed()

azure-pipelines/dotnet.yml

@@ -1,14 +1,16 @@
+parameters:
+- name: test_secrets
+  type: object
+  default: { }
+
 steps:
 
-- script: dotnet build --no-restore -c $(BuildConfiguration) /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/build.binlog"
+- script: dotnet build --no-restore -c $(BuildConfiguration) -p:Platform="$(BuildPlatform)" /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/build.binlog"
   displayName: dotnet build
 
-- script: dotnet pack --no-build -c $(BuildConfiguration) /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/pack.binlog"
+- script: dotnet pack --no-build -c $(BuildConfiguration) -p:Platform="$(BuildPlatform)" /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/pack.binlog"
   displayName: dotnet pack
 
-- pwsh: ./tools/Emulators.ps1
-  displayName: Start storage emulators
-
 - task: DotNetCoreCLI@2
   displayName: dotnet test -f net472
   inputs:
@@ -25,13 +27,17 @@ steps:
     arguments: --no-build -c $(BuildConfiguration) -f netcoreapp2.1 --filter "TestCategory!=FailsInCloudTest" -v n /p:CollectCoverage=true --settings "$(Build.Repository.LocalPath)/azure-pipelines/$(Agent.OS).runsettings"
     workingDirectory: test/IronPigeon.Tests
     testRunTitle: netcoreapp2.1-$(Agent.JobName)
+  env:
+    ${{ parameters.test_secrets }}
 
 - task: DotNetCoreCLI@2
   displayName: dotnet test -f netcoreapp3.1
   inputs:
     command: test
-    arguments: --no-build -c $(BuildConfiguration) -f netcoreapp3.1 --filter "TestCategory!=FailsInCloudTest" -v n /p:CollectCoverage=true --settings "$(Build.Repository.LocalPath)/azure-pipelines/$(Agent.OS).runsettings"
+    arguments: --no-build -c $(BuildConfiguration) -p:Platform="$(BuildPlatform)" -f netcoreapp3.1 --filter "TestCategory!=FailsInCloudTest" -v n /p:CollectCoverage=true --settings "$(Build.Repository.LocalPath)/azure-pipelines/$(Agent.OS).runsettings"
     testRunTitle: netcoreapp3.1-$(Agent.JobName)
+  env:
+    ${{ parameters.test_secrets }}
 
 - powershell: azure-pipelines/variables/_pipelines.ps1
   failOnStderr: true

azure-pipelines/publish-codecoverage.yml

@@ -3,10 +3,10 @@ steps:
   artifact: coverageResults-Windows
   displayName: Download Windows code coverage results
   continueOnError: true
-# - download: current
-#   artifact: coverageResults-Linux
-#   displayName: Download Linux code coverage results
-#   continueOnError: true
+- download: current
+  artifact: coverageResults-Linux
+  displayName: Download Linux code coverage results
+  continueOnError: true
 # - download: current
 #   artifact: coverageResults-macOS
 #   displayName: Download macOS code coverage results

init.ps1

@@ -48,7 +48,11 @@ if (!$NoPrerequisites) {
         Exit 3010
     }
 
-    & "$PSScriptRoot\tools\Install-AzureCosmosDBEmulator.ps1" -InstallLocality $InstallLocality
+    if ($IsLinux -or $IsMacOS) {
+        Write-Warning "Azure Cosmos DB Emulator is only compatible with Windows. Use an Azure Cosmos DB Table API account for local development instead. Set CosmosDBConnectionString to the connection string as an environment variable."
+    } else {
+        & "$PSScriptRoot\tools\Install-AzureCosmosDBEmulator.ps1" -InstallLocality $InstallLocality
+    }
 
     # The procdump tool and env var is required for dotnet test to collect hang/crash dumps of tests.
     # But it only works on Windows.

src/IronPigeon.Relay/AzureStorage.cs

@@ -20,9 +20,13 @@ public AzureStorage(IConfiguration configuration)
             this.CosmosDBConnectionString = configuration.GetValue<string>("CosmosDBConnectionString") ?? throw new InvalidOperationException("Missing CosmosDBConnectionString in configuration.");
             this.TableCloudStorageAccount = CloudStorageAccount.Parse(this.CosmosDBConnectionString);
             this.TableClient = this.TableCloudStorageAccount.CreateCloudTableClient();
-            this.InboxTable = this.TableClient.GetTableReference("Inboxes");
-            this.InboxItemContainer = new BlobContainerClient(this.AzureStorageConnectionString, "inbox-items");
-            this.PayloadBlobsContainer = new BlobContainerClient(this.AzureStorageConnectionString, "payloads");
+
+            // We want to isolate tests for a particular run, particularly in case they're using a shared resource like a test environment in Azure.
+            string? tableNameSuffix = Environment.GetEnvironmentVariable("BUILD_BUILDID");
+
+            this.InboxTable = this.TableClient.GetTableReference("Inboxes" + tableNameSuffix);
+            this.InboxItemContainer = new BlobContainerClient(this.AzureStorageConnectionString, "inbox-items" + tableNameSuffix);
+            this.PayloadBlobsContainer = new BlobContainerClient(this.AzureStorageConnectionString, "payloads" + tableNameSuffix);
         }
 
         /// <summary>

src/IronPigeon.Relay/Startup.cs

@@ -34,7 +34,8 @@ public static async Task InitializeDatabasesAsync(CancellationToken cancellation
         {
             IConfigurationBuilder cb = new ConfigurationBuilder()
                .SetBasePath(Directory.GetCurrentDirectory())
-               .AddJsonFile("appsettings.json");
+               .AddJsonFile("appsettings.json")
+               .AddEnvironmentVariables();
             var azureStorage = new AzureStorage(cb.Build());
 
             await azureStorage.InboxTable.CreateIfNotExistsAsync(cancellationToken);

test/IronPigeon.Tests/EllipticCurveCryptoPatterns.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading;
@@ -43,10 +44,11 @@ public EllipticCurveCryptoPatterns(ITestOutputHelper logger)
 #if NETCOREAPP2_1
     [Fact(Skip = "Fails on .NET Core 2.1 due to a crypto bug.")]
 #else
-    [Fact]
+    [SkippableFact]
 #endif
     public async Task ECAsymmetricSigningAndEncryption()
     {
+        Skip.IfNot(RuntimeInformation.IsOSPlatform(OSPlatform.Windows));
         using var bob = new ECDsaCng(521);
         var bobPublic = CngKey.Import(bob.Key.Export(CngKeyBlobFormat.EccPublicBlob), CngKeyBlobFormat.EccPublicBlob);
         using var alice = new ECDsaCng(521);

test/IronPigeon.Tests/IronPigeon.Tests.csproj

@@ -14,6 +14,7 @@
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
     <PackageReference Include="Moq" Version="4.14.6" />
+    <PackageReference Include="Xunit.SkippableFact" Version="1.4.13" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\IronPigeon\IronPigeon.csproj" />

tools/Emulators.ps1

@@ -26,7 +26,9 @@ if (Test-Path "$env:ProgramW6432\Azure Cosmos DB Emulator\PSModules\Microsoft.Az
     Write-Error "Azure CosmosDB Emulator not found."
 }
 
+Write-Host "Starting Cosmos DB Emulator..."
 Start-CosmosDbEmulator
 
+Write-Host "Starting Azurite in another window..."
 if (!(Test-Path $EmulatorStoragePath)) { New-Item -ItemType Directory -Path $EmulatorStoragePath | Out-Null }
 cmd /c start yarn run azurite-blob -l $EmulatorStoragePath -d $EmulatorLogPath

tools/Emulators.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+EmulatorStoragePath=obj/azurite
+EmulatorLogPath=obj/azurite.log
+
+if [ ! -d "$EmulatorStoragePath" ]
+then
+    mkdir $EmulatorStoragePath
+fi
+
+yarn run azurite -l $EmulatorStoragePath -d $EmulatorLogPath

tools/Install-AzureCosmosDBEmulator.ps1

@@ -4,6 +4,11 @@ Param (
     [string]$InstallLocality='machine'
 )
 
+if ($IsLinux -or $IsMacOS) {
+    Write-Error "Azure Cosmos DB Emulator is only compatible with Windows. Use an Azure Cosmos DB Table API account for local development instead. Set CosmosDBConnectionString to the connection string as an environment variable."
+    exit 2
+}
+
 if ((Test-Path "$env:ProgramW6432\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator") -or (Test-Path "$env:ProgramFiles\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator")) {
     Write-Verbose "Azure Cosmos DB Emulator is already installed."
     exit
@@ -22,4 +27,5 @@ $installerPath = Join-Path $binaryToolsPath AzureCosmosDBEmulatorInstaller.msi
 Write-Verbose "Downloading installer to $installerPath"
 (New-Object System.Net.WebClient).DownloadFile('https://aka.ms/cosmosdb-emulator', $installerPath)
 
+Write-Host "Installing Azure Cosmos DB Emulator..."
 cmd /c start /wait "$InstallerPath" /passive