diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index dfdd86e127..b3725fcb9b 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -57,6 +57,8 @@ $actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." "Asset cache miss; downloading from .*" +"Downloading .*" +"Successfully downloaded .*." "Successfully stored .* to .*." ) -join "`n" @@ -82,26 +84,116 @@ if (-not ($actual -match $expected)) { # Testing asset caching && x-block-orgin promises when --debug is passed (enabled) Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) -$actual = $actual -replace "`r`n", "`n" - -# Define the regex pattern that accounts for multiline input -$expectedPattern = "(?s)" + - ".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.\.?" + - ".*\[DEBUG\] Asset caching is enabled\..*" - -if (-not ($actual -match $expectedPattern)) { +if (-not ($actual.Contains("[DEBUG] External asset downloads are blocked (x-block-origin is enabled)") -and $actual.Contains("[DEBUG] Asset caching is enabled."))) { throw "Failure: couldn't find expected debug promises (asset caching enabled + x-block-origin enabled)" } # Testing asset caching && x-block-orgin promises when --debug is passed (disabled) Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug")) +if (-not ($actual.Contains("[DEBUG] External asset downloads are allowed (x-block-origin is disabled)") -and $actual.Contains("[DEBUG] Asset cache is not configured"))) { + throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)" +} + +# azurl (no), x-block-origin (no), asset-cache (n/a), download (fail) +# Expected: Download failure message, nothing about asset caching +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html")) +if (-not ($actual.Contains("error: https://localhost:1234/foobar.html: curl failed to download with exit code 7"))) { + throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)" +} + +#azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch) +#Expected: Download message with the "you might need to configure a proxy" message and with expected/actual sha +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com")) +if (-not ($actual.Contains("Failed to download example3.html.") -and + $actual.Contains("If you are using a proxy, please ensure your proxy settings are correct.") -and + $actual.Contains("error: File does not have the expected hash:") -and + $actual.Contains("url: https://example.com") -and + $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and + $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) { + throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch)" +} + +# azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed) +# Expected: Download success message, nothing about asset caching +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com")) +if (-not ($actual.Contains("Downloading example3.html") -and + $actual.Contains("Successfully downloaded example3.html."))) { + throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)" +} + +# azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a) +# Expected: Download failure message, nothing about asset caching, x-block-origin complaint +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=clear;x-block-origin")) +if (-not ($actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) { + throw "Failure: azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)" +} + +# azurl (yes), x-block-origin (no), asset-cache (miss), download (fail) +# Expected: Download failure message, asset cache named, nothing about x-block-origin +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +if (-not ($actual.Contains("Asset cache miss; downloading from https://localhost:1234/foobar.html") -and + $actual.Contains("Downloading example3.html") -and + $actual.Contains("error: file://$AssetCache") -and + $actual.Contains("curl failed to download with exit code 37"))) { + throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (fail)" +} + +# azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a) +# Expected: Download success message, asset cache named, nothing about x-block-origin +Refresh-TestRoot $actual = $actual -replace "`r`n", "`n" +Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) { + throw "Failure: azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)" +} + +# azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch) +# Expected: Download message with "you might need to configure a proxy" and expected/actual sha +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and + $actual.Contains("Downloading example3.html") -and + $actual.Contains("error: file://$AssetCache") -and + $actual.Contains("curl failed to download with exit code 37") -and + $actual.Contains("error: File does not have the expected hash:") -and + $actual.Contains("url: https://example.com") -and + $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and + $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) { + throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch)" +} -$expectedPattern = "(?s)" + - ".*\[DEBUG\] External asset downloads are allowed \(x-block-origin is disabled\)\.\.\.?" + - ".*\[DEBUG\] Asset cache is not configured.*" +# azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed) +# Expected: Download success message, asset cache upload, nothing about x-block-origin +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and + $actual.Contains("Downloading example3.html") -and + $actual.Contains("Successfully downloaded example3.html.") -and + $actual.Contains("Successfully stored example3.html to file://$AssetCache"))) { + throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)" +} -if (-not ($actual -match $expectedPattern)) { - throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)" +# azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a) +# Expected: Download failure message, which asset cache was tried, x-block-origin complaint +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin")) +if (-not ($actual.Contains("Asset cache miss for example3.html and downloads are blocked by x-block-origin.") -and + $actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) { + throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a)" +} + +# azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a) +# Expected: Download success message, asset cache named, nothing about x-block-origin +Refresh-TestRoot +Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin")) +if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) { + throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)" } diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 82d4e5c8ae..3131761a91 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -248,6 +248,11 @@ DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when a DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}; downloaded from: {url}") DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}") +DECLARE_MESSAGE(AssetCacheMissBlockOrigin, + (msg::path), + "x-block-origin is a vcpkg term. Do not translate", + "Asset cache miss for {path} and downloads are blocked by x-block-origin.") +DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}.") DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), @@ -1046,6 +1051,22 @@ DECLARE_MESSAGE(DownloadFailedStatusCode, (msg::url, msg::value), "{value} is an HTTP status code", "{url}: failed: status code {value}") +DECLARE_MESSAGE(DownloadFailedProxySettings, + (msg::path, msg::url), + "", + "Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are " + "correct.\nPossible causes are:\n" + "1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable " + "to `https//address:port`.\nThis is not correct, because `https://` prefix " + "claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\n" + "Try setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n" + "2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings " + "set by your proxy software. See, {url}\n" + "The value set by your proxy might be wrong, or have same `https://` prefix issue.\n" + "3. Your proxy's remote server is our of service.\n" + "If you've tried directly download the link, and believe this is not a temporay download server " + "failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n" + "to report this upstream download server failure.") DECLARE_MESSAGE(DownloadingPortableToolVersionX, (msg::tool_name, msg::version), "", @@ -1240,10 +1261,6 @@ DECLARE_MESSAGE(MissingShaVariable, (), "{{sha}} should not be translated", "The {{sha}} variable must be used in the template if other variables are used.") -DECLARE_MESSAGE(AssetCacheMissBlockOrigin, - (msg::path), - "x-block-origin is a vcpkg term. Do not translate", - "Asset cache miss for {path} and downloads are blocked by x-block-origin.") DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":") DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.") DECLARE_MESSAGE(FailedToFindPortFeature, diff --git a/locales/messages.json b/locales/messages.json index d26ca78f6f..d33ef52815 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -595,12 +595,16 @@ "_DownloadFailedCurl.comment": "An example of {url} is https://github.com/microsoft/vcpkg. An example of {exit_code} is 127.", "DownloadFailedHashMismatch": "File does not have the expected hash:\nurl: {url}\nFile: {path}\nExpected hash: {expected}\nActual hash: {actual}", "_DownloadFailedHashMismatch.comment": "{expected} and {actual} are SHA512 hashes in hex format. An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.", + "DownloadFailedProxySettings": "Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are correct.\nPossible causes are:\n1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to `https//address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\nTry setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software. See, {url}\nThe value set by your proxy might be wrong, or have same `https://` prefix issue.\n3. Your proxy's remote server is our of service.\nIf you've tried directly download the link, and believe this is not a temporay download server failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\nto report this upstream download server failure.", + "_DownloadFailedProxySettings.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadFailedRetrying": "Download failed -- retrying after {value}ms", "_DownloadFailedRetrying.comment": "{value} is a number of milliseconds", "DownloadFailedStatusCode": "{url}: failed: status code {value}", "_DownloadFailedStatusCode.comment": "{value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadRootsDir": "Downloads directory (default: {env_var})", "_DownloadRootsDir.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", + "DownloadSuccesful": "Successfully downloaded {path}.", + "_DownloadSuccesful.comment": "An example of {path} is /foo/bar.", "DownloadWinHttpError": "{url}: {system_api} failed with exit code {exit_code}", "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadedSources": "Downloaded sources for {spec}", diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 13ead15feb..56e61316c5 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -905,6 +905,8 @@ namespace vcpkg MessageSink& progress_sink) const { std::vector errors; + bool block_origin_enabled = m_config.m_block_origin; + if (urls.size() == 0) { if (auto hash = sha512.get()) @@ -936,6 +938,14 @@ namespace vcpkg msg::url = replace_secrets(read_url, m_config.m_secrets)); return read_url; } + else if (block_origin_enabled) + { + msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename()); + } + else + { + msg::println(msgAssetCacheMiss, msg::url = urls[0]); + } } else if (auto script = m_config.m_script.get()) { @@ -985,16 +995,20 @@ namespace vcpkg } } - if (!m_config.m_block_origin) + if (block_origin_enabled) + { + msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename()); + } + else { if (urls.size() != 0) { + msg::println(msgDownloadingUrl, msg::url = download_path.filename()); auto maybe_url = try_download_file( fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); if (auto url = maybe_url.get()) { - m_config.m_read_url_template.has_value() ? msg::println(msgAssetCacheMiss, msg::url = urls[0]) - : msg::println(msgDownloadingUrl, msg::url = urls[0]); + msg::println(msgDownloadSuccesful, msg::path = download_path.filename()); if (auto hash = sha512.get()) { @@ -1010,17 +1024,14 @@ namespace vcpkg return *url; } + else + { + msg::println(msgDownloadFailedProxySettings, + msg::path = download_path.filename(), + msg::url = "https://github.com/microsoft/vcpkg-tool/pull/77"); + } } } - // Asset cache is not configured and x-block-origin enabled - if (m_config.m_read_url_template.has_value()) - { - msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename()); - } - else - { - msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename()); - } for (LocalizedString& error : errors) {