Clarify asset caching output (#1409)

microsoft · Jul 6, 2024 · 9eb99f7 · 9eb99f7
1 parent 68e3b2f
commit 9eb99f7
Show file tree

Hide file tree

Showing 8 changed files with 175 additions and 23 deletions.
diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1
@@ -1,6 +1,107 @@
 . $PSScriptRoot/../end-to-end-tests-prelude.ps1
 
+# Testing x-script
 Refresh-TestRoot
 Run-Vcpkg -TestArgs ($commonArgs + @('fetch', 'cmake'))
 Run-Vcpkg -TestArgs ($commonArgs + @("install", "vcpkg-test-x-script", "--x-binarysource=clear", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-script,$TestScriptAssetCacheExe {url} {sha512} {dst};x-block-origin"))
 Throw-IfFailed
+
+$env:VCPKG_FORCE_DOWNLOADED_BINARIES = "ON"
+
+# Testing asset cache miss (not configured) + x-block-origin enabled
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot"))
+$actual = $actual -replace "`r`n", "`n"
+
+$expected = @(
+"A suitable version of .* was not found \(required v[0-9\.]+\)."
+"error: Missing .* and downloads are blocked by x-block-origin."
+) -join "`n"
+
+if (-not ($actual -match $expected)) {
+    throw "Failure: asset cache miss (not configured) + x-block-origin enabled"
+}
+
+# Testing asset cache miss (not configured) + x-block-origin disabled
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot"))
+$actual = $actual -replace "`r`n", "`n"
+
+$expected = @(
+"A suitable version of .* was not found \(required v[0-9\.]+\)."
+"Downloading .*."
+) -join "`n"
+
+if (-not ($actual -match $expected)) {
+    throw "Failure: asset cache miss (not configured) + x-block-origin disabled"
+}
+
+# Testing asset cache miss (configured) + x-block-origin enabled
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot"))
+$actual = $actual -replace "`r`n", "`n"
+
+$expected = @(
+"A suitable version of .* was not found \(required v[0-9\.]+\)."
+"Asset cache miss for .* and downloads are blocked by x-block-origin"
+) -join "`n"
+
+if (-not ($actual -match $expected)) {
+    throw "Failure: asset cache miss (configured) + x-block-origin disabled"
+}
+
+# Testing asset cache miss (configured) + x-block-origin disabled
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot"))
+$actual = $actual -replace "`r`n", "`n"
+
+$expected = @(
+"A suitable version of .* was not found \(required v[0-9\.]+\)."
+"Asset cache miss; downloading from .*"
+"Successfully stored .* to .*."
+) -join "`n"
+
+if (-not ($actual -match $expected)) {
+    throw "Failure: asset cache miss (configured) + x-block-origin disabled"
+}
+
+# Testing asset cache hit
+Refresh-Downloads
+Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port'))
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot"))
+$actual = $actual -replace "`r`n", "`n"
+
+$expected = @(
+"A suitable version of .* was not found \(required v[0-9\.]+\)."
+"Asset cache hit for .*; downloaded from: .*"
+) -join "`n"
+
+if (-not ($actual -match $expected)) {
+    throw "Failure: asset cache hit"
+}
+
+# Testing asset caching && x-block-orgin promises when --debug is passed (enabled)
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug"))
+$actual = $actual -replace "`r`n", "`n"
+
+# Define the regex pattern that accounts for multiline input
+$expectedPattern = "(?s)" +
+                   ".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.\.?" +
+                   ".*\[DEBUG\] Asset caching is enabled\..*"
+
+if (-not ($actual -match $expectedPattern)) {
+    throw "Failure: couldn't find expected debug promises (asset caching enabled + x-block-origin enabled)"
+}
+
+# Testing asset caching && x-block-orgin promises when --debug is passed (disabled)
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug"))
+$actual = $actual -replace "`r`n", "`n"
+
+$expectedPattern = "(?s)" +
+                   ".*\[DEBUG\] External asset downloads are allowed \(x-block-origin is disabled\)\.\.\.?" +
+                   ".*\[DEBUG\] Asset cache is not configured.*"
+
+if (-not ($actual -match $expectedPattern)) {
+    throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)"
+}
diff --git a/azure-pipelines/end-to-end-tests-prelude.ps1 b/azure-pipelines/end-to-end-tests-prelude.ps1
@@ -6,6 +6,9 @@ $NuGetRoot = Join-Path $TestingRoot 'nuget'
 $NuGetRoot2 = Join-Path $TestingRoot 'nuget2'
 $ArchiveRoot = Join-Path $TestingRoot 'archives'
 $VersionFilesRoot = Join-Path $TestingRoot 'version-test'
+$DownloadsRoot = Join-Path $TestingRoot 'downloads'
+$AssetCache = Join-Path $TestingRoot 'asset-cache'
+
 $directoryArgs = @(
     "--x-buildtrees-root=$buildtreesRoot",
     "--x-install-root=$installRoot",
@@ -34,6 +37,13 @@ function Refresh-TestRoot {
     Remove-Item -Recurse -Force $TestingRoot -ErrorAction SilentlyContinue
     New-Item -ItemType Directory -Force $TestingRoot | Out-Null
     New-Item -ItemType Directory -Force $NuGetRoot | Out-Null
+    New-Item -ItemType Directory -Force $DownloadsRoot | Out-Null
+    New-Item -ItemType Directory -Force $AssetCache | Out-Null
+}
+
+function Refresh-Downloads{
+    Remove-Item -Recurse -Force $DownloadsRoot -ErrorAction SilentlyContinue
+    New-Item -ItemType Directory -Force $DownloadsRoot | Out-Null
 }
 
 function Write-Stack {

diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h
@@ -92,6 +92,9 @@ namespace vcpkg
                                           const Path& file_to_put,
                                           StringView sha512) const;
 
+        bool get_block_origin() const;
+        bool asset_cache_configured() const;
+
     private:
         DownloadManagerConfig m_config;
     };

diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h
@@ -233,10 +233,14 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when
 DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts")
 DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts")
 DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts")
+DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}; downloaded from: {url}")
+DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}")
+DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}")
 DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments,
                 (msg::value),
                 "{value} is a asset caching provider name such as azurl, clear, or x-block-origin",
                 "unexpected arguments: '{value}' does not accept arguments")
+DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path, msg::url), "", "Successfully stored {path} to {url}.")
 DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'")
 DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string")
 DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features")
@@ -1032,10 +1036,7 @@ DECLARE_MESSAGE(DownloadFailedStatusCode,
 DECLARE_MESSAGE(DownloadingPortableToolVersionX,
                 (msg::tool_name, msg::version),
                 "",
-                "A suitable version of {tool_name} was not found (required v{version}) Downloading "
-                "portable {tool_name} {version}...")
-DECLARE_MESSAGE(DownloadingTool, (msg::tool_name, msg::url, msg::path), "", "Downloading {tool_name}...\n{url}->{path}")
-DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}")
+                "A suitable version of {tool_name} was not found (required v{version}).")
 DECLARE_MESSAGE(DownloadWinHttpError,
                 (msg::system_api, msg::exit_code, msg::url),
                 "",
@@ -1206,7 +1207,14 @@ DECLARE_MESSAGE(FailedToDeleteInsideDueToFile,
                 "printed after this",
                 "failed to remove_all_inside({value}) due to {path}: ")
 DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the current commit:")
-DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, (), "", "Failed to download from mirror set")
+DECLARE_MESSAGE(MissingAssetBlockOrigin,
+                (msg::path),
+                "x-block-origin is a vcpkg term. Do not translate",
+                "Missing {path} and downloads are blocked by x-block-origin.")
+DECLARE_MESSAGE(AssetCacheMissBlockOrigin,
+                (msg::path),
+                "x-block-origin is a vcpkg term. Do not translate",
+                "Asset cache miss for {path} and downloads are blocked by x-block-origin.")
 DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":")
 DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.")
 DECLARE_MESSAGE(FailedToFindPortFeature,
@@ -1255,7 +1263,7 @@ DECLARE_MESSAGE(FailedToRunToolToDetermineVersion,
                 "Additional information, such as the command line output, if any, will be appended on "
                 "the line after this message",
                 "Failed to run \"{path}\" to determine the {tool_name} version.")
-DECLARE_MESSAGE(FailedToStoreBackToMirror, (), "", "failed to store back to mirror:")
+DECLARE_MESSAGE(FailedToStoreBackToMirror, (msg::path, msg::url), "", "Failed to store {path} to {url}.")
 DECLARE_MESSAGE(FailedToStoreBinaryCache, (msg::path), "", "Failed to store binary cache {path}")
 DECLARE_MESSAGE(FailedToTakeFileSystemLock, (), "", "Failed to take the filesystem lock")
 DECLARE_MESSAGE(FailedVendorAuthentication,
@@ -2681,7 +2689,7 @@ DECLARE_MESSAGE(UnexpectedArgument,
 DECLARE_MESSAGE(
     UnexpectedAssetCacheProvider,
     (),
-    "",
+    "'x-azurl', 'x-script', 'clear' are valid source types. Do not translate",
     "unknown asset provider type: valid source types are 'x-azurl', 'x-script', 'x-block-origin', and 'clear'")
 DECLARE_MESSAGE(UnexpectedByteSize,
                 (msg::expected, msg::actual),

diff --git a/locales/messages.json b/locales/messages.json
@@ -167,8 +167,16 @@
   "ArtifactsSwitchWindows": "Forces host detection to Windows when acquiring artifacts",
   "ArtifactsSwitchX64": "Forces host detection to x64 when acquiring artifacts",
   "ArtifactsSwitchX86": "Forces host detection to x86 when acquiring artifacts",
+  "AssetCacheHit": "Asset cache hit for {path}; downloaded from: {url}",
+  "_AssetCacheHit.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.",
+  "AssetCacheMiss": "Asset cache miss; downloading from {url}",
+  "_AssetCacheMiss.comment": "An example of {url} is https://github.com/microsoft/vcpkg.",
+  "AssetCacheMissBlockOrigin": "Asset cache miss for {path} and downloads are blocked by x-block-origin.",
+  "_AssetCacheMissBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.",
   "AssetCacheProviderAcceptsNoArguments": "unexpected arguments: '{value}' does not accept arguments",
   "_AssetCacheProviderAcceptsNoArguments.comment": "{value} is a asset caching provider name such as azurl, clear, or x-block-origin",
+  "AssetCacheSuccesfullyStored": "Successfully stored {path} to {url}.",
+  "_AssetCacheSuccesfullyStored.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.",
   "AssetSourcesArg": "Asset caching sources. See 'vcpkg help assetcaching'",
   "AttemptingToSetBuiltInBaseline": "attempting to set builtin-baseline in vcpkg.json while overriding the default-registry in vcpkg-configuration.json.\nthe default-registry from vcpkg-configuration.json will be used.",
   "AuthenticationMayRequireManualAction": "One or more {vendor} credential providers requested manual action. Add the binary source 'interactive' to allow interactivity.",
@@ -593,10 +601,8 @@
   "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
   "DownloadedSources": "Downloaded sources for {spec}",
   "_DownloadedSources.comment": "An example of {spec} is zlib:x64-windows.",
-  "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}) Downloading portable {tool_name} {version}...",
+  "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}).",
   "_DownloadingPortableToolVersionX.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.",
-  "DownloadingTool": "Downloading {tool_name}...\n{url}->{path}",
-  "_DownloadingTool.comment": "An example of {tool_name} is aria2. An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.",
   "DownloadingUrl": "Downloading {url}",
   "_DownloadingUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.",
   "DownloadingVcpkgStandaloneBundle": "Downloading standalone bundle {version}.",
@@ -716,7 +722,6 @@
   "FailedToDeleteInsideDueToFile": "failed to remove_all_inside({value}) due to {path}: ",
   "_FailedToDeleteInsideDueToFile.comment": "{value} is the parent path of {path} we tried to delete; the underlying Windows error message is printed after this An example of {path} is /foo/bar.",
   "FailedToDetermineCurrentCommit": "Failed to determine the current commit:",
-  "FailedToDownloadFromMirrorSet": "Failed to download from mirror set",
   "FailedToExtract": "Failed to extract \"{path}\":",
   "_FailedToExtract.comment": "An example of {path} is /foo/bar.",
   "FailedToFetchRepo": "Failed to fetch {url}.",
@@ -751,7 +756,8 @@
   "_FailedToParseVersionXML.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.",
   "FailedToRunToolToDetermineVersion": "Failed to run \"{path}\" to determine the {tool_name} version.",
   "_FailedToRunToolToDetermineVersion.comment": "Additional information, such as the command line output, if any, will be appended on the line after this message An example of {tool_name} is aria2. An example of {path} is /foo/bar.",
-  "FailedToStoreBackToMirror": "failed to store back to mirror:",
+  "FailedToStoreBackToMirror": "Failed to store {path} to {url}.",
+  "_FailedToStoreBackToMirror.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.",
   "FailedToStoreBinaryCache": "Failed to store binary cache {path}",
   "_FailedToStoreBinaryCache.comment": "An example of {path} is /foo/bar.",
   "FailedToTakeFileSystemLock": "Failed to take the filesystem lock",
@@ -1127,6 +1133,8 @@
   "MissingAndroidHomeDir": "ANDROID_NDK_HOME directory does not exist: {path}",
   "_MissingAndroidHomeDir.comment": "An example of {path} is /foo/bar.",
   "MissingArgFormatManifest": "format-manifest was passed --convert-control without '--all'.\nThis doesn't do anything: control files passed explicitly are converted automatically.",
+  "MissingAssetBlockOrigin": "Missing {path} and downloads are blocked by x-block-origin.",
+  "_MissingAssetBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.",
   "MissingClosingParen": "missing closing )",
   "MissingDependency": "Package {spec} is installed, but dependency {package_name} is not.",
   "_MissingDependency.comment": "An example of {spec} is zlib:x64-windows. An example of {package_name} is zlib.",
@@ -1426,6 +1434,7 @@
   "UnexpectedArgument": "unexpected argument: {option}",
   "_UnexpectedArgument.comment": "Argument is literally what the user passed on the command line. An example of {option} is editable.",
   "UnexpectedAssetCacheProvider": "unknown asset provider type: valid source types are 'x-azurl', 'x-script', 'x-block-origin', and 'clear'",
+  "_UnexpectedAssetCacheProvider.comment": "'x-azurl', 'x-script', 'clear' are valid source types. Do not translate",
   "UnexpectedByteSize": "Expected {expected} bytes to be written, but {actual} were written.",
   "_UnexpectedByteSize.comment": "{expected} is the expected byte size and {actual} is the actual byte size.",
   "UnexpectedCharExpectedCloseBrace": "Unexpected character; expected property or close brace",