From fa1f708f367cd1396dbc70752e58f540fcfd9f70 Mon Sep 17 00:00:00 2001 From: "John Erickson (from Dev Box)" Date: Fri, 8 Dec 2023 15:05:30 -0800 Subject: [PATCH 1/2] ForceCacheMissPercentage --- README.md | 1 + src/Common.Tests/PluginSettingsTests.cs | 7 +++++++ src/Common/MSBuildCachePluginBase.cs | 20 ++++++++++++++++++- src/Common/PluginSettings.cs | 2 ++ .../Microsoft.MSBuildCache.Common.targets | 2 ++ src/Local/MSBuildCacheLocalPlugin.cs | 1 - 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f92a99c..0507323 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ These settings are common across all plugins, although different implementations | `$(MSBuildCacheMaxConcurrentCacheContentOperations)` | `int` | 64 | The maximum number of cache operations to perform concurrently | | `$(MSBuildCacheLocalCacheRootPath)` | `string` | "\MSBuildCache" (in repo's drive root) | Base directory to use for the local cache. | | `$(MSBuildCacheLocalCacheSizeInMegabytes)` | `int` | 102400 (100 GB) | The maximum size in megabytes of the local cache | +| `$(ForceCacheMissPercentage)` | `int` | 0 | Force a cache miss a percentage of the time. Useful for testing to simulate natural cache misses. | | `$(MSBuildCacheIgnoredInputPatterns)` | `Glob[]` | | Files which are part of the repo which should be ignored for cache invalidation | | `$(MSBuildCacheIgnoredOutputPatterns)` | `Glob[]` | `*.assets.cache; *assemblyreference.cache` | Files to ignore for output collection into the cache. Note that if output are ignored, the replayed cache entry will not have these files. This should be used for intermediate outputs which are not properly portable | | `$(MSBuildCacheIdenticalDuplicateOutputPatterns)` | `Glob[]` | | Files to allow duplicate outputs, with identical content, across projects. Projects which produce files at the same path with differing content will still produce an error. Note: this setting should be used sparingly as it impacts performance | diff --git a/src/Common.Tests/PluginSettingsTests.cs b/src/Common.Tests/PluginSettingsTests.cs index dc7c598..5c4327a 100644 --- a/src/Common.Tests/PluginSettingsTests.cs +++ b/src/Common.Tests/PluginSettingsTests.cs @@ -94,6 +94,13 @@ public void LocalCacheSizeInMegabytesSetting() pluginSettings => pluginSettings.LocalCacheSizeInMegabytes, new[] { 123u, 456u, 789u }); + [TestMethod] + public void ForceCacheMissBitMaskSetting() + => TestBasicSetting( + nameof(PluginSettings.ForceCacheMissPercentage), + pluginSettings => pluginSettings.ForceCacheMissPercentage, + new[] { 0u, 10u, 100u }); + private static IEnumerable GlobTestData { get diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index 10bf2bc..18e54f8 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -11,6 +11,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Threading; @@ -105,7 +106,8 @@ static MSBuildCachePluginBase() => nameof(_fileAccessRepository), nameof(_cacheClient), nameof(_ignoredOutputPatterns), - nameof(_identicalDuplicateOutputPatterns) + nameof(_identicalDuplicateOutputPatterns), + nameof(Settings) )] protected bool Initialized { get; private set; } @@ -350,6 +352,22 @@ private async Task GetCacheResultInnerAsync(BuildRequestData buildR return CacheResult.IndicateNonCacheHit(CacheResultType.CacheMiss); } + if (Settings.ForceCacheMissPercentage != 0) + { + // use a hash of the project id so that it is repeatable +#pragma warning disable CA1850 // ComputeHash is not in net472 + using SHA256 hasher = SHA256.Create(); + byte[] projectHash = hasher.ComputeHash(Encoding.UTF8.GetBytes(nodeContext.Id)); + uint hashInt = BitConverter.ToUInt32(projectHash, 0); +#pragma warning restore CA1850 + if ((hashInt % 100) < Settings.ForceCacheMissPercentage) + { + logger.LogMessage($"Forcing an otherwise 'cache hit' to be a 'cache miss' because of ForceCacheMissPercentage."); + Interlocked.Increment(ref _cacheMissCount); + return CacheResult.IndicateNonCacheHit(CacheResultType.CacheMiss); + } + } + CheckForDuplicateOutputs(logger, nodeBuildResult.Outputs, nodeContext); await FinishNodeAsync(logger, nodeContext, pathSet, nodeBuildResult); diff --git a/src/Common/PluginSettings.cs b/src/Common/PluginSettings.cs index 54fe4b4..96e5cd3 100644 --- a/src/Common/PluginSettings.cs +++ b/src/Common/PluginSettings.cs @@ -81,6 +81,8 @@ public string LocalCacheRootPath public uint LocalCacheSizeInMegabytes { get; init; } = 102400; // 100GB + public uint ForceCacheMissPercentage { get; init; } + public IReadOnlyCollection IgnoredInputPatterns { get; init; } = Array.Empty(); public IReadOnlyCollection IgnoredOutputPatterns { get; init; } = Array.Empty(); diff --git a/src/Common/build/Microsoft.MSBuildCache.Common.targets b/src/Common/build/Microsoft.MSBuildCache.Common.targets index 75cc593..24fbff1 100644 --- a/src/Common/build/Microsoft.MSBuildCache.Common.targets +++ b/src/Common/build/Microsoft.MSBuildCache.Common.targets @@ -10,6 +10,7 @@ $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheMaxConcurrentCacheContentOperations $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheLocalCacheRootPath $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheLocalCacheSizeInMegabytes + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheForceCacheMissPercentage $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheIgnoredInputPatterns $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheIgnoredOutputPatterns $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheIdenticalDuplicateOutputPatterns @@ -29,6 +30,7 @@ $(MSBuildCacheMaxConcurrentCacheContentOperations) $(MSBuildCacheLocalCacheRootPath) $(MSBuildCacheLocalCacheSizeInMegabytes) + $(MSBuildCacheForceCacheMissPercentage) $(MSBuildCacheIgnoredInputPatterns) $(MSBuildCacheIgnoredOutputPatterns) $(MSBuildCacheIdenticalDuplicateOutputPatterns) diff --git a/src/Local/MSBuildCacheLocalPlugin.cs b/src/Local/MSBuildCacheLocalPlugin.cs index 640bd53..c0a9730 100644 --- a/src/Local/MSBuildCacheLocalPlugin.cs +++ b/src/Local/MSBuildCacheLocalPlugin.cs @@ -38,7 +38,6 @@ protected override async Task CreateCacheClientAsync(PluginLoggerB #pragma warning restore CA2000 // Dispose objects before losing scope Context context = new(cacheLogger); - #pragma warning disable CA2000 // Dispose objects before losing scope. Expected to be disposed by TwoLevelCache LocalCache cache = LocalCacheFactory.Create(cacheLogger, Settings.LocalCacheRootPath, Settings.LocalCacheSizeInMegabytes); #pragma warning restore CA2000 // Dispose objects before losing scope From d4ee000d04fc03936293d224a8961826f6a8e71f Mon Sep 17 00:00:00 2001 From: John Erickson Date: Tue, 12 Dec 2023 09:01:26 -0800 Subject: [PATCH 2/2] PR feedback --- src/Common.Tests/PluginSettingsTests.cs | 2 +- src/Common/MSBuildCachePluginBase.cs | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Common.Tests/PluginSettingsTests.cs b/src/Common.Tests/PluginSettingsTests.cs index 5c4327a..73e2ff5 100644 --- a/src/Common.Tests/PluginSettingsTests.cs +++ b/src/Common.Tests/PluginSettingsTests.cs @@ -95,7 +95,7 @@ public void LocalCacheSizeInMegabytesSetting() new[] { 123u, 456u, 789u }); [TestMethod] - public void ForceCacheMissBitMaskSetting() + public void ForceCacheMissPercentageSetting() => TestBasicSetting( nameof(PluginSettings.ForceCacheMissPercentage), pluginSettings => pluginSettings.ForceCacheMissPercentage, diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index 18e54f8..a60c677 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -355,11 +355,15 @@ private async Task GetCacheResultInnerAsync(BuildRequestData buildR if (Settings.ForceCacheMissPercentage != 0) { // use a hash of the project id so that it is repeatable -#pragma warning disable CA1850 // ComputeHash is not in net472 + +#if NETFRAMEWORK using SHA256 hasher = SHA256.Create(); byte[] projectHash = hasher.ComputeHash(Encoding.UTF8.GetBytes(nodeContext.Id)); +#else + byte[] projectHash = SHA256.HashData(Encoding.UTF8.GetBytes(nodeContext.Id)); +#endif + uint hashInt = BitConverter.ToUInt32(projectHash, 0); -#pragma warning restore CA1850 if ((hashInt % 100) < Settings.ForceCacheMissPercentage) { logger.LogMessage($"Forcing an otherwise 'cache hit' to be a 'cache miss' because of ForceCacheMissPercentage.");