diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index fcf1792bbe..dfdd86e127 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -1,6 +1,107 @@ . $PSScriptRoot/../end-to-end-tests-prelude.ps1 +# Testing x-script Refresh-TestRoot Run-Vcpkg -TestArgs ($commonArgs + @('fetch', 'cmake')) Run-Vcpkg -TestArgs ($commonArgs + @("install", "vcpkg-test-x-script", "--x-binarysource=clear", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-script,$TestScriptAssetCacheExe {url} {sha512} {dst};x-block-origin")) Throw-IfFailed + +$env:VCPKG_FORCE_DOWNLOADED_BINARIES = "ON" + +# Testing asset cache miss (not configured) + x-block-origin enabled +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" + +$expected = @( +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"error: Missing .* and downloads are blocked by x-block-origin." +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Failure: asset cache miss (not configured) + x-block-origin enabled" +} + +# Testing asset cache miss (not configured) + x-block-origin disabled +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" + +$expected = @( +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"Downloading .*." +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Failure: asset cache miss (not configured) + x-block-origin disabled" +} + +# Testing asset cache miss (configured) + x-block-origin enabled +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" + +$expected = @( +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"Asset cache miss for .* and downloads are blocked by x-block-origin" +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Failure: asset cache miss (configured) + x-block-origin disabled" +} + +# Testing asset cache miss (configured) + x-block-origin disabled +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" + +$expected = @( +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"Asset cache miss; downloading from .*" +"Successfully stored .* to .*." +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Failure: asset cache miss (configured) + x-block-origin disabled" +} + +# Testing asset cache hit +Refresh-Downloads +Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port')) +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" + +$expected = @( +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"Asset cache hit for .*; downloaded from: .*" +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Failure: asset cache hit" +} + +# Testing asset caching && x-block-orgin promises when --debug is passed (enabled) +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) +$actual = $actual -replace "`r`n", "`n" + +# Define the regex pattern that accounts for multiline input +$expectedPattern = "(?s)" + + ".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.\.?" + + ".*\[DEBUG\] Asset caching is enabled\..*" + +if (-not ($actual -match $expectedPattern)) { + throw "Failure: couldn't find expected debug promises (asset caching enabled + x-block-origin enabled)" +} + +# Testing asset caching && x-block-orgin promises when --debug is passed (disabled) +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug")) +$actual = $actual -replace "`r`n", "`n" + +$expectedPattern = "(?s)" + + ".*\[DEBUG\] External asset downloads are allowed \(x-block-origin is disabled\)\.\.\.?" + + ".*\[DEBUG\] Asset cache is not configured.*" + +if (-not ($actual -match $expectedPattern)) { + throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)" +} diff --git a/azure-pipelines/end-to-end-tests-prelude.ps1 b/azure-pipelines/end-to-end-tests-prelude.ps1 index e5b6156bec..e580c72266 100644 --- a/azure-pipelines/end-to-end-tests-prelude.ps1 +++ b/azure-pipelines/end-to-end-tests-prelude.ps1 @@ -6,6 +6,9 @@ $NuGetRoot = Join-Path $TestingRoot 'nuget' $NuGetRoot2 = Join-Path $TestingRoot 'nuget2' $ArchiveRoot = Join-Path $TestingRoot 'archives' $VersionFilesRoot = Join-Path $TestingRoot 'version-test' +$DownloadsRoot = Join-Path $TestingRoot 'downloads' +$AssetCache = Join-Path $TestingRoot 'asset-cache' + $directoryArgs = @( "--x-buildtrees-root=$buildtreesRoot", "--x-install-root=$installRoot", @@ -34,6 +37,13 @@ function Refresh-TestRoot { Remove-Item -Recurse -Force $TestingRoot -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force $TestingRoot | Out-Null New-Item -ItemType Directory -Force $NuGetRoot | Out-Null + New-Item -ItemType Directory -Force $DownloadsRoot | Out-Null + New-Item -ItemType Directory -Force $AssetCache | Out-Null +} + +function Refresh-Downloads{ + Remove-Item -Recurse -Force $DownloadsRoot -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force $DownloadsRoot | Out-Null } function Write-Stack { diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index a361e24f76..93b4163ee5 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -92,6 +92,9 @@ namespace vcpkg const Path& file_to_put, StringView sha512) const; + bool get_block_origin() const; + bool asset_cache_configured() const; + private: DownloadManagerConfig m_config; }; diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index fb00872070..6e066f52bc 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -233,10 +233,14 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") +DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}; downloaded from: {url}") +DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}") +DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") +DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path, msg::url), "", "Successfully stored {path} to {url}.") DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'") DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") @@ -1032,10 +1036,7 @@ DECLARE_MESSAGE(DownloadFailedStatusCode, DECLARE_MESSAGE(DownloadingPortableToolVersionX, (msg::tool_name, msg::version), "", - "A suitable version of {tool_name} was not found (required v{version}) Downloading " - "portable {tool_name} {version}...") -DECLARE_MESSAGE(DownloadingTool, (msg::tool_name, msg::url, msg::path), "", "Downloading {tool_name}...\n{url}->{path}") -DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") + "A suitable version of {tool_name} was not found (required v{version}).") DECLARE_MESSAGE(DownloadWinHttpError, (msg::system_api, msg::exit_code, msg::url), "", @@ -1206,7 +1207,14 @@ DECLARE_MESSAGE(FailedToDeleteInsideDueToFile, "printed after this", "failed to remove_all_inside({value}) due to {path}: ") DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the current commit:") -DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, (), "", "Failed to download from mirror set") +DECLARE_MESSAGE(MissingAssetBlockOrigin, + (msg::path), + "x-block-origin is a vcpkg term. Do not translate", + "Missing {path} and downloads are blocked by x-block-origin.") +DECLARE_MESSAGE(AssetCacheMissBlockOrigin, + (msg::path), + "x-block-origin is a vcpkg term. Do not translate", + "Asset cache miss for {path} and downloads are blocked by x-block-origin.") DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":") DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.") DECLARE_MESSAGE(FailedToFindPortFeature, @@ -1255,7 +1263,7 @@ DECLARE_MESSAGE(FailedToRunToolToDetermineVersion, "Additional information, such as the command line output, if any, will be appended on " "the line after this message", "Failed to run \"{path}\" to determine the {tool_name} version.") -DECLARE_MESSAGE(FailedToStoreBackToMirror, (), "", "failed to store back to mirror:") +DECLARE_MESSAGE(FailedToStoreBackToMirror, (msg::path, msg::url), "", "Failed to store {path} to {url}.") DECLARE_MESSAGE(FailedToStoreBinaryCache, (msg::path), "", "Failed to store binary cache {path}") DECLARE_MESSAGE(FailedToTakeFileSystemLock, (), "", "Failed to take the filesystem lock") DECLARE_MESSAGE(FailedVendorAuthentication, @@ -2681,7 +2689,7 @@ DECLARE_MESSAGE(UnexpectedArgument, DECLARE_MESSAGE( UnexpectedAssetCacheProvider, (), - "", + "'x-azurl', 'x-script', 'clear' are valid source types. Do not translate", "unknown asset provider type: valid source types are 'x-azurl', 'x-script', 'x-block-origin', and 'clear'") DECLARE_MESSAGE(UnexpectedByteSize, (msg::expected, msg::actual), diff --git a/locales/messages.json b/locales/messages.json index eb061dd990..fdd5df0bf9 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -167,8 +167,16 @@ "ArtifactsSwitchWindows": "Forces host detection to Windows when acquiring artifacts", "ArtifactsSwitchX64": "Forces host detection to x64 when acquiring artifacts", "ArtifactsSwitchX86": "Forces host detection to x86 when acquiring artifacts", + "AssetCacheHit": "Asset cache hit for {path}; downloaded from: {url}", + "_AssetCacheHit.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheMiss": "Asset cache miss; downloading from {url}", + "_AssetCacheMiss.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheMissBlockOrigin": "Asset cache miss for {path} and downloads are blocked by x-block-origin.", + "_AssetCacheMissBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "AssetCacheProviderAcceptsNoArguments": "unexpected arguments: '{value}' does not accept arguments", "_AssetCacheProviderAcceptsNoArguments.comment": "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", + "AssetCacheSuccesfullyStored": "Successfully stored {path} to {url}.", + "_AssetCacheSuccesfullyStored.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "AssetSourcesArg": "Asset caching sources. See 'vcpkg help assetcaching'", "AttemptingToSetBuiltInBaseline": "attempting to set builtin-baseline in vcpkg.json while overriding the default-registry in vcpkg-configuration.json.\nthe default-registry from vcpkg-configuration.json will be used.", "AuthenticationMayRequireManualAction": "One or more {vendor} credential providers requested manual action. Add the binary source 'interactive' to allow interactivity.", @@ -593,10 +601,8 @@ "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadedSources": "Downloaded sources for {spec}", "_DownloadedSources.comment": "An example of {spec} is zlib:x64-windows.", - "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}) Downloading portable {tool_name} {version}...", + "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}).", "_DownloadingPortableToolVersionX.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", - "DownloadingTool": "Downloading {tool_name}...\n{url}->{path}", - "_DownloadingTool.comment": "An example of {tool_name} is aria2. An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.", "DownloadingUrl": "Downloading {url}", "_DownloadingUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadingVcpkgStandaloneBundle": "Downloading standalone bundle {version}.", @@ -716,7 +722,6 @@ "FailedToDeleteInsideDueToFile": "failed to remove_all_inside({value}) due to {path}: ", "_FailedToDeleteInsideDueToFile.comment": "{value} is the parent path of {path} we tried to delete; the underlying Windows error message is printed after this An example of {path} is /foo/bar.", "FailedToDetermineCurrentCommit": "Failed to determine the current commit:", - "FailedToDownloadFromMirrorSet": "Failed to download from mirror set", "FailedToExtract": "Failed to extract \"{path}\":", "_FailedToExtract.comment": "An example of {path} is /foo/bar.", "FailedToFetchRepo": "Failed to fetch {url}.", @@ -751,7 +756,8 @@ "_FailedToParseVersionXML.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", "FailedToRunToolToDetermineVersion": "Failed to run \"{path}\" to determine the {tool_name} version.", "_FailedToRunToolToDetermineVersion.comment": "Additional information, such as the command line output, if any, will be appended on the line after this message An example of {tool_name} is aria2. An example of {path} is /foo/bar.", - "FailedToStoreBackToMirror": "failed to store back to mirror:", + "FailedToStoreBackToMirror": "Failed to store {path} to {url}.", + "_FailedToStoreBackToMirror.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "FailedToStoreBinaryCache": "Failed to store binary cache {path}", "_FailedToStoreBinaryCache.comment": "An example of {path} is /foo/bar.", "FailedToTakeFileSystemLock": "Failed to take the filesystem lock", @@ -1127,6 +1133,8 @@ "MissingAndroidHomeDir": "ANDROID_NDK_HOME directory does not exist: {path}", "_MissingAndroidHomeDir.comment": "An example of {path} is /foo/bar.", "MissingArgFormatManifest": "format-manifest was passed --convert-control without '--all'.\nThis doesn't do anything: control files passed explicitly are converted automatically.", + "MissingAssetBlockOrigin": "Missing {path} and downloads are blocked by x-block-origin.", + "_MissingAssetBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "MissingClosingParen": "missing closing )", "MissingDependency": "Package {spec} is installed, but dependency {package_name} is not.", "_MissingDependency.comment": "An example of {spec} is zlib:x64-windows. An example of {package_name} is zlib.", @@ -1426,6 +1434,7 @@ "UnexpectedArgument": "unexpected argument: {option}", "_UnexpectedArgument.comment": "Argument is literally what the user passed on the command line. An example of {option} is editable.", "UnexpectedAssetCacheProvider": "unknown asset provider type: valid source types are 'x-azurl', 'x-script', 'x-block-origin', and 'clear'", + "_UnexpectedAssetCacheProvider.comment": "'x-azurl', 'x-script', 'clear' are valid source types. Do not translate", "UnexpectedByteSize": "Expected {expected} bytes to be written, but {actual} were written.", "_UnexpectedByteSize.comment": "{expected} is the expected byte size and {actual} is the actual byte size.", "UnexpectedCharExpectedCloseBrace": "Unexpected character; expected property or close brace", diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index d179e8078e..9f2e44cfd3 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -580,7 +580,9 @@ namespace vcpkg msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = url, msg::value = code); } } - + msg::println(msgAssetCacheSuccesfullyStored, + msg::path = file.filename(), + msg::url = replace_secrets(url.to_string(), secrets)); return res; } @@ -715,7 +717,6 @@ namespace vcpkg fs.create_directories(dir, VCPKG_LINE_INFO); const auto sanitized_url = replace_secrets(url, secrets); - msg::println(msgDownloadingUrl, msg::url = sanitized_url); static auto s = WinHttpSession::make(sanitized_url).value_or_exit(VCPKG_LINE_INFO); for (size_t trials = 0; trials < 4; ++trials) { @@ -882,6 +883,10 @@ namespace vcpkg return s_headers; } + bool DownloadManager::get_block_origin() const { return m_config.m_block_origin; } + + bool DownloadManager::asset_cache_configured() const { return m_config.m_read_url_template.has_value(); } + void DownloadManager::download_file(const Filesystem& fs, const std::string& url, View headers, @@ -926,6 +931,9 @@ namespace vcpkg errors, progress_sink)) { + msg::println(msgAssetCacheHit, + msg::path = download_path.filename(), + msg::url = replace_secrets(read_url, m_config.m_secrets)); return read_url; } } @@ -985,12 +993,17 @@ namespace vcpkg fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); if (auto url = maybe_url.get()) { + m_config.m_read_url_template.has_value() ? msg::println(msgAssetCacheMiss, msg::url = urls[0]) + : msg::println(msgDownloadingUrl, msg::url = urls[0]); + if (auto hash = sha512.get()) { auto maybe_push = put_file_to_mirror(fs, download_path, *hash); if (!maybe_push) { - msg::println_warning(msgFailedToStoreBackToMirror); + msg::println_warning(msgFailedToStoreBackToMirror, + msg::path = download_path.filename(), + msg::url = replace_secrets(download_path.c_str(), m_config.m_secrets)); msg::println(maybe_push.error()); } } @@ -999,7 +1012,16 @@ namespace vcpkg } } } - msg::println_error(msgFailedToDownloadFromMirrorSet); + // Asset cache is not configured and x-block-origin enabled + if (m_config.m_read_url_template.has_value()) + { + msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename()); + } + else + { + msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename()); + } + for (LocalizedString& error : errors) { msg::println(error); diff --git a/src/vcpkg/tools.cpp b/src/vcpkg/tools.cpp index 9181632fda..0fb43745d7 100644 --- a/src/vcpkg/tools.cpp +++ b/src/vcpkg/tools.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -677,12 +678,6 @@ namespace vcpkg const auto download_path = downloads / tool_data.download_subpath; if (!fs.exists(download_path, IgnoreErrors{})) { - status_sink.println(Color::none, - msgDownloadingTool, - msg::tool_name = tool_data.name, - msg::url = tool_data.url, - msg::path = download_path); - downloader->download_file(fs, tool_data.url, {}, download_path, tool_data.sha512, null_sink); } else diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index b16494f649..900b92b838 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -656,9 +656,13 @@ namespace vcpkg , community_triplets(filesystem.almost_canonical(triplets / "community", VCPKG_LINE_INFO)) { Debug::print("Using vcpkg-root: ", root, '\n'); - Debug::print("Using scripts-root: ", scripts, '\n'); Debug::print("Using builtin-registry: ", builtin_registry_versions, '\n'); Debug::print("Using downloads-root: ", downloads, '\n'); + m_pimpl->m_download_manager->get_block_origin() + ? Debug::println("External asset downloads are blocked (x-block-origin is enabled)..") + : Debug::println("External asset downloads are allowed (x-block-origin is disabled)..."); + m_pimpl->m_download_manager->asset_cache_configured() ? Debug::println("Asset caching is enabled.") + : Debug::println("Asset cache is not configured."); { const auto config_path = m_pimpl->m_config_dir / "vcpkg-configuration.json";