Merged PR 13558: Merge from github

dotnet · Mar 12, 2021 · e313f5d · e313f5d
2 parents 14b57cf + 25be3b5
commit e313f5d
Show file tree

Hide file tree

Showing 11 changed files with 166 additions and 105 deletions.
diff --git a/NuGet.config b/NuGet.config
@@ -2,10 +2,13 @@
 <configuration>
   <packageSources>
     <clear />
+    <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
+    <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
     <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
-    <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
+    <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
     <add key="dotnet3.1" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json" />
     <add key="dotnet3.1-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json" />
+    <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
   </packageSources>
   <disabledPackageSources />
 </configuration>
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -65,21 +65,21 @@ Note: if the Uri is a new place, you will need to add a subscription from that p
       <Sha>c2e8c9f71737d87a95610851587e2a1eaf18c91c</Sha>
     </Dependency>
     <!-- Arcade -->
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.21072.7">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.21158.9">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>620462dd065490620c03a80cea8e251004f6bf21</Sha>
+      <Sha>ab42d248928f65dd174a2383f3256e68d9379bb7</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.GenFacades" Version="1.0.0-beta.21072.7">
+    <Dependency Name="Microsoft.DotNet.GenFacades" Version="1.0.0-beta.21158.9">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>620462dd065490620c03a80cea8e251004f6bf21</Sha>
+      <Sha>ab42d248928f65dd174a2383f3256e68d9379bb7</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.21072.7">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.21158.9">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>620462dd065490620c03a80cea8e251004f6bf21</Sha>
+      <Sha>ab42d248928f65dd174a2383f3256e68d9379bb7</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="2.4.1-beta.21072.7">
+    <Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="2.4.1-beta.21158.9">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>620462dd065490620c03a80cea8e251004f6bf21</Sha>
+      <Sha>ab42d248928f65dd174a2383f3256e68d9379bb7</Sha>
     </Dependency>
     <!-- Keep these dependencies at the bottom of ToolsetDependencies, else they will be picked as the parent for CoherentParentDependencies -->
     <Dependency Name="Microsoft.NETCore.App.Ref" Version="3.1.0" Pinned="true">

diff --git a/eng/Versions.props b/eng/Versions.props
@@ -37,8 +37,8 @@
   </PropertyGroup>
   <!-- Arcade -->
   <PropertyGroup>
-    <MicrosoftDotNetGenFacadesPackageVersion>1.0.0-beta.21072.7</MicrosoftDotNetGenFacadesPackageVersion>
-    <MicrosoftDotNetXUnitExtensionsPackageVersion>2.4.1-beta.21072.7</MicrosoftDotNetXUnitExtensionsPackageVersion>
+    <MicrosoftDotNetGenFacadesPackageVersion>1.0.0-beta.21158.9</MicrosoftDotNetGenFacadesPackageVersion>
+    <MicrosoftDotNetXUnitExtensionsPackageVersion>2.4.1-beta.21158.9</MicrosoftDotNetXUnitExtensionsPackageVersion>
   </PropertyGroup>
   <!-- Below have no corresponding entries in Versions.Details.XML because they are not updated via Maestro -->
   <!-- XUnit-related (not extensions) -->

diff --git a/eng/common/build.ps1 b/eng/common/build.ps1
@@ -22,40 +22,44 @@ Param(
   [switch] $ci,
   [switch] $prepareMachine,
   [switch] $help,
+  [string] $runtimeSourceFeed = "",
+  [string] $runtimeSourceFeedKey = "",
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
 )
 
 . $PSScriptRoot\tools.ps1
 
 function Print-Usage() {
     Write-Host "Common settings:"
-    Write-Host "  -configuration <value>  Build configuration: 'Debug' or 'Release' (short: -c)"
-    Write-Host "  -platform <value>       Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild"
-    Write-Host "  -verbosity <value>      Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
-    Write-Host "  -binaryLog              Output binary log (short: -bl)"
-    Write-Host "  -help                   Print help and exit"
+    Write-Host "  -configuration <value>        Build configuration: 'Debug' or 'Release' (short: -c)"
+    Write-Host "  -platform <value>             Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild"
+    Write-Host "  -verbosity <value>            Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
+    Write-Host "  -binaryLog                    Output binary log (short: -bl)"
+    Write-Host "  -help                         Print help and exit"
     Write-Host ""
 
     Write-Host "Actions:"
-    Write-Host "  -restore                Restore dependencies (short: -r)"
-    Write-Host "  -build                  Build solution (short: -b)"
-    Write-Host "  -rebuild                Rebuild solution"
-    Write-Host "  -deploy                 Deploy built VSIXes"
-    Write-Host "  -deployDeps             Deploy dependencies (e.g. VSIXes for integration tests)"
-    Write-Host "  -test                   Run all unit tests in the solution (short: -t)"
-    Write-Host "  -integrationTest        Run all integration tests in the solution"
-    Write-Host "  -performanceTest        Run all performance tests in the solution"
-    Write-Host "  -pack                   Package build outputs into NuGet packages and Willow components"
-    Write-Host "  -sign                   Sign build outputs"
-    Write-Host "  -publish                Publish artifacts (e.g. symbols)"
+    Write-Host "  -restore                      Restore dependencies (short: -r)"
+    Write-Host "  -build                        Build solution (short: -b)"
+    Write-Host "  -rebuild                      Rebuild solution"
+    Write-Host "  -deploy                       Deploy built VSIXes"
+    Write-Host "  -deployDeps                   Deploy dependencies (e.g. VSIXes for integration tests)"
+    Write-Host "  -test                         Run all unit tests in the solution (short: -t)"
+    Write-Host "  -integrationTest              Run all integration tests in the solution"
+    Write-Host "  -performanceTest              Run all performance tests in the solution"
+    Write-Host "  -pack                         Package build outputs into NuGet packages and Willow components"
+    Write-Host "  -sign                         Sign build outputs"
+    Write-Host "  -publish                      Publish artifacts (e.g. symbols)"
     Write-Host ""
 
     Write-Host "Advanced settings:"
-    Write-Host "  -projects <value>       Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
-    Write-Host "  -ci                     Set when running on CI server"
-    Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
-    Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
-    Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+    Write-Host "  -projects <value>             Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
+    Write-Host "  -ci                           Set when running on CI server"
+    Write-Host "  -prepareMachine               Prepare machine for CI run, clean up processes after build"
+    Write-Host "  -warnAsError <value>          Sets warnaserror msbuild parameter ('true' or 'false')"
+    Write-Host "  -msbuildEngine <value>        Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+    Write-Host "  -runtimeSourceFeed <value>    Alternate feed source for restoring .NET Runtimes and SDKs"
+    Write-Host "  -runtimeSourceFeedKey <value> Key value for non-public values of runtimeSourceFeed"
     Write-Host ""
 
     Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -75,7 +79,7 @@ function InitializeCustomToolset {
 }
 
 function Build {
-  $toolsetBuildProj = InitializeToolset
+  $toolsetBuildProj = InitializeToolset -runtimeSourceFeed $runtimeSourceFeed -runtimeSourceFeedKey $runtimeSourceFeedKey
   InitializeCustomToolset
 
   $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" }

diff --git a/eng/common/build.sh b/eng/common/build.sh
@@ -29,11 +29,14 @@ usage()
   echo ""
 
   echo "Advanced settings:"
-  echo "  --projects <value>       Project or solution file(s) to build"
-  echo "  --ci                     Set when running on CI server"
-  echo "  --prepareMachine         Prepare machine for CI run, clean up processes after build"
-  echo "  --nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
-  echo "  --warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
+  echo "  --projects <value>             Project or solution file(s) to build"
+  echo "  --ci                           Set when running on CI server"
+  echo "  --prepareMachine               Prepare machine for CI run, clean up processes after build"
+  echo "  --nodeReuse <value>            Sets nodereuse msbuild parameter ('true' or 'false')"
+  echo "  --warnAsError <value>          Sets warnaserror msbuild parameter ('true' or 'false')"
+  echo "  --runtimeSourceFeed <value>    Alternate (fallback) source for .NET Runtime and SDK installation"
+  echo "  --runtimeSourceFeedKey <value> Credentials (if needed) for authenticating to runtimeSourceFeed"
+
   echo ""
   echo "Command line arguments not listed above are passed thru to msbuild."
   echo "Arguments can also be passed in with a single hyphen."
@@ -73,6 +76,9 @@ configuration='Debug'
 prepare_machine=false
 verbosity='minimal'
 
+runtimeSourceFeed=''
+runtimeSourceFeedKey=''
+
 properties=''
 
 while [[ $# > 0 ]]; do
@@ -141,6 +147,14 @@ while [[ $# > 0 ]]; do
       node_reuse=$2
       shift
       ;;
+    -runtimesourcefeed)
+      shift
+      runtimeSourceFeed="$1"
+      ;;
+    -runtimesourcefeedkey)
+      shift
+      runtimeSourceFeedKey="$1"
+      ;;
     *)
       properties="$properties $1"
       ;;
@@ -166,7 +180,7 @@ function InitializeCustomToolset {
 }
 
 function Build {
-  InitializeToolset
+  InitializeToolset $runtimeSourceFeed $runtimeSourceFeedKey
   InitializeCustomToolset
 
   if [[ ! -z "$projects" ]]; then

diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1
@@ -64,7 +64,6 @@ function SetupCredProvider {
   }
 
   if (($endpoints | Measure-Object).Count -gt 0) {
-      # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
       $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
 
      # Create the environment variables the AzDo way

diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh
@@ -62,7 +62,6 @@ function SetupCredProvider {
   endpoints+=']'
 
   if [ ${#endpoints} -gt 2 ]; then 
-      # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
       local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
 
       echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"

diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
@@ -11,6 +11,12 @@ $ErrorActionPreference = "Stop"
 Set-StrictMode -Version 2.0
 $LASTEXITCODE = 0
 
+# `tools.ps1` checks $ci to perform some actions. Since the SDL
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+. $PSScriptRoot\..\tools.ps1
+
 # Don't display the console progress UI - it's a huge perf hit
 $ProgressPreference = 'SilentlyContinue'
 
@@ -21,19 +27,9 @@ $uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cf
 $zipFile = "$WorkingDirectory/gdn.zip"
 
 Add-Type -AssemblyName System.IO.Compression.FileSystem
-$gdnFolder = (Join-Path $WorkingDirectory ".gdn")
-Try
-{
-  # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead
-  Write-Host "Downloading gdn folder from internal config repostiory..."
-  Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile
-  if (Test-Path $gdnFolder) {
-    # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case)
-    Remove-Item -Force -Recurse $gdnFolder
-  }
-  [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory)
-  Write-Host $gdnFolder
-} Catch [System.Net.WebException] {
+$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
+
+try {
   # if the folder does not exist, we'll do a guardian init and push it to the remote repository
   Write-Host "Initializing Guardian..."
   Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel"
@@ -48,4 +44,10 @@ Try
     Write-Error "Guardian baseline failed with exit code $LASTEXITCODE."
   }
   & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Initialize gdn folder"
+  ExitWithExitCode 0
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
 }
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
@@ -96,7 +96,7 @@ function Exec-Process([string]$command, [string]$commandArgs) {
   }
 }
 
-function InitializeDotNetCli([bool]$install) {
+function InitializeDotNetCli([bool]$install, [string] $runtimeSourceFeed, [string] $runtimeSourceFeedKey) {
   if (Test-Path variable:global:_DotNetInstallDir) {
     return $global:_DotNetInstallDir
   }
@@ -138,7 +138,7 @@ function InitializeDotNetCli([bool]$install) {
 
     if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) {
       if ($install) {
-        InstallDotNetSdk $dotnetRoot $dotnetSdkVersion
+        InstallDotNetSdk -dotnetRoot $dotnetRoot -version $dotnetSdkVersion -runtimeSourceFeed $runtimeSourceFeed -runtimeSourceFeedKey $runtimeSourceFeedKey
       } else {
         Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'"
         ExitWithExitCode 1
@@ -207,8 +207,8 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
   return $installScript
 }
 
-function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") {
-  InstallDotNet $dotnetRoot $version $architecture
+function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtimeSourceFeed, [string] $runtimeSourceFeedKey ) {
+  InstallDotNet -dotnetRoot $dotnetRoot -version $version -architecture $architecture -skipNonVersionedFiles $false -runtimeSourceFeed $runtimeSourceFeed -runtimeSourceFeedKey $runtimeSourceFeedKey
 }
 
 function InstallDotNet([string] $dotnetRoot, 
@@ -233,10 +233,9 @@ function InstallDotNet([string] $dotnetRoot,
     & $installScript @installParameters
   }
   catch {
-    Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from public location."
-
-    # Only the runtime can be installed from a custom [private] location.
-    if ($runtime -and ($runtimeSourceFeed -or $runtimeSourceFeedKey)) {
+    # If we fail, retry from a custom (possibly private) location.
+    if ($runtimeSourceFeed -or $runtimeSourceFeedKey) {
+      Write-Host "Failed to install dotnet runtime '$runtime' version '$version' from public location; trying specified alternate feed credentials"
       if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
 
       if ($runtimeSourceFeedKey) {
@@ -253,6 +252,7 @@ function InstallDotNet([string] $dotnetRoot,
         ExitWithExitCode 1
       }
     } else {
+      Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' version '$version' from public location."
       ExitWithExitCode 1
     }
   }
@@ -439,7 +439,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){
   return $vsInfo[0]
 }
 
-function InitializeBuildTool() {
+function InitializeBuildTool([string] $runtimeSourceFeed, [string] $runtimeSourceFeedKey) {
   if (Test-Path variable:global:_BuildTool) {
     return $global:_BuildTool
   }
@@ -451,7 +451,7 @@ function InitializeBuildTool() {
   # Initialize dotnet cli if listed in 'tools'
   $dotnetRoot = $null
   if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") {
-    $dotnetRoot = InitializeDotNetCli -install:$restore
+    $dotnetRoot = InitializeDotNetCli -install:$restore -runtimeSourceFeed $runtimeSourceFeed -runtimeSourceFeedKey $runtimeSourceFeedKey
   }
 
   if ($msbuildEngine -eq "dotnet") {
@@ -523,7 +523,8 @@ function InitializeNativeTools() {
   }
 }
 
-function InitializeToolset() {
+function InitializeToolset([string] $runtimeSourceFeed, [string] $runtimeSourceFeedKey) 
+{
   if (Test-Path variable:global:_ToolsetBuildProj) {
     return $global:_ToolsetBuildProj
   }
@@ -545,7 +546,7 @@ function InitializeToolset() {
     ExitWithExitCode 1
   }
 
-  $buildTool = InitializeBuildTool
+  $buildTool = InitializeBuildTool -runtimeSourceFeed $runtimeSourceFeed -runtimeSourceFeedKey $runtimeSourceFeedKey
 
   $proj = Join-Path $ToolsetDir "restore.proj"
   $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" }