From 327578c1139d2271dc9254cb8ce903db2ae14c66 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 5 Sep 2024 11:34:24 -0600 Subject: [PATCH 01/18] HACK: Remove `Console.InputEncoding` use --- src/Common/SourceControl/Git.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/SourceControl/Git.cs b/src/Common/SourceControl/Git.cs index dbfdc76..97b5223 100644 --- a/src/Common/SourceControl/Git.cs +++ b/src/Common/SourceControl/Git.cs @@ -15,8 +15,10 @@ public static class Git private static readonly object InputEncodingLock = new object(); #endif +#if NET // UTF8 - NO BOM private static readonly Encoding InputEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); +#endif public static async Task BranchNameAsync(PluginLoggerBase logger, string repoRoot) { @@ -57,7 +59,7 @@ public static async Task RunAsync( Encoding originalConsoleInputEncoding = Console.InputEncoding; try { - Console.InputEncoding = InputEncoding; + //Console.InputEncoding = InputEncoding; process.Start(); } From da81a8b9e84fabbea7bed18d98e7e73b17785aba Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 5 Sep 2024 12:12:20 -0600 Subject: [PATCH 02/18] HACK: ProjectCachePlugin2 --- src/Common/build/Microsoft.MSBuildCache.Common.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/build/Microsoft.MSBuildCache.Common.targets b/src/Common/build/Microsoft.MSBuildCache.Common.targets index 75cc593..cfc6e4e 100644 --- a/src/Common/build/Microsoft.MSBuildCache.Common.targets +++ b/src/Common/build/Microsoft.MSBuildCache.Common.targets @@ -23,7 +23,7 @@ - + $(MSBuildCacheLogDirectory) $(MSBuildCacheCacheUniverse) $(MSBuildCacheMaxConcurrentCacheContentOperations) @@ -39,7 +39,7 @@ $(MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns) $(MSBuildCacheAllowProcessCloseAfterProjectFinishProcessPatterns) $(MSBuildCacheGlobalPropertiesToIgnore) - + From ecfc627d1c839ed90cc75c0aaab039e15e7e410f Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 5 Sep 2024 12:59:16 -0600 Subject: [PATCH 03/18] fixup! HEAD~2 --- src/Common/SourceControl/Git.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/SourceControl/Git.cs b/src/Common/SourceControl/Git.cs index 97b5223..4494567 100644 --- a/src/Common/SourceControl/Git.cs +++ b/src/Common/SourceControl/Git.cs @@ -65,7 +65,7 @@ public static async Task RunAsync( } finally { - Console.InputEncoding = originalConsoleInputEncoding; + //Console.InputEncoding = originalConsoleInputEncoding; } } #else From 010d9238908bd1eeb750ff07a108d8934e6976f1 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 5 Sep 2024 14:07:55 -0600 Subject: [PATCH 04/18] Ignore more properties --- src/Common/build/Microsoft.MSBuildCache.Common.targets | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Common/build/Microsoft.MSBuildCache.Common.targets b/src/Common/build/Microsoft.MSBuildCache.Common.targets index cfc6e4e..145dd7d 100644 --- a/src/Common/build/Microsoft.MSBuildCache.Common.targets +++ b/src/Common/build/Microsoft.MSBuildCache.Common.targets @@ -20,6 +20,9 @@ $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheAllowProcessCloseAfterProjectFinishProcessPatterns $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheGlobalPropertiesToIgnore + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCachePackageName + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCachePackageVersion + $(MSBuildCacheGlobalPropertiesToIgnore);IsGraphBuild From 5d1f7d9c97871c2841d9d81b65984d925537149b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 5 Sep 2024 14:50:28 -0600 Subject: [PATCH 05/18] Activate real item under a switch --- src/Common/build/Microsoft.MSBuildCache.Common.targets | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/build/Microsoft.MSBuildCache.Common.targets b/src/Common/build/Microsoft.MSBuildCache.Common.targets index 145dd7d..a70eb0a 100644 --- a/src/Common/build/Microsoft.MSBuildCache.Common.targets +++ b/src/Common/build/Microsoft.MSBuildCache.Common.targets @@ -43,6 +43,7 @@ $(MSBuildCacheAllowProcessCloseAfterProjectFinishProcessPatterns) $(MSBuildCacheGlobalPropertiesToIgnore) + From 9f826275a08d04f126a532bb98e964d30033270c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 5 Sep 2024 15:25:32 -0600 Subject: [PATCH 06/18] Ignore more --- src/Common/build/Microsoft.MSBuildCache.Common.targets | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Common/build/Microsoft.MSBuildCache.Common.targets b/src/Common/build/Microsoft.MSBuildCache.Common.targets index a70eb0a..92d3de1 100644 --- a/src/Common/build/Microsoft.MSBuildCache.Common.targets +++ b/src/Common/build/Microsoft.MSBuildCache.Common.targets @@ -23,6 +23,8 @@ $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCachePackageName $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCachePackageVersion $(MSBuildCacheGlobalPropertiesToIgnore);IsGraphBuild + $(MSBuildCacheGlobalPropertiesToIgnore);UseHostCompilerIfAvailable + From dfc7ca81c01b361aca11a8f608fef7f91956164c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 5 Sep 2024 15:25:57 -0600 Subject: [PATCH 07/18] Allow matching on a subset of targets --- src/Common/MSBuildCachePluginBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index 0704964..d909e27 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -355,7 +355,7 @@ private async Task GetCacheResultInnerAsync(BuildRequestData buildR nodeContext.SetStartTime(); - if (!nodeContext.TargetNames.SetEquals(buildRequest.TargetNames)) + if (!nodeContext.TargetNames.IsSupersetOf(buildRequest.TargetNames)) { logger.LogMessage($"`TargetNames` does not match for {nodeContext.Id}. `{string.Join(";", nodeContext.TargetNames)}` vs `{string.Join(";", buildRequest.TargetNames)}`."); return CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable); From ae3fbdd040f923c36904b30f2c6682265056f845 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 5 Sep 2024 15:26:28 -0600 Subject: [PATCH 08/18] Add DebuggerDisplay to `NodeDescriptor` --- src/Common/NodeDescriptor.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Common/NodeDescriptor.cs b/src/Common/NodeDescriptor.cs index a584e1e..0eed8c2 100644 --- a/src/Common/NodeDescriptor.cs +++ b/src/Common/NodeDescriptor.cs @@ -3,9 +3,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics; namespace Microsoft.MSBuildCache; +[DebuggerDisplay($"{{{nameof(DebuggerDisplay)}, nq}}")] public readonly struct NodeDescriptor : IEquatable { private readonly string _projectFullPath; @@ -23,6 +25,8 @@ public NodeDescriptor(string projectFullPath, SortedDictionary f /// public IReadOnlyDictionary FilteredGlobalProperties => _filteredGlobalProperties; + private string DebuggerDisplay => $"{_projectFullPath} ({_filteredGlobalProperties.Count})"; + public bool Equals(NodeDescriptor other) { if (!_projectFullPath.Equals(other._projectFullPath, StringComparison.OrdinalIgnoreCase)) From 6aa826f180d333b80eb7abf6b8b04159606697f5 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 6 Sep 2024 10:27:53 -0600 Subject: [PATCH 09/18] Ignore empty TargetFramework properties --- src/Common/NodeDescriptorFactory.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Common/NodeDescriptorFactory.cs b/src/Common/NodeDescriptorFactory.cs index 0198984..5b0546c 100644 --- a/src/Common/NodeDescriptorFactory.cs +++ b/src/Common/NodeDescriptorFactory.cs @@ -42,6 +42,11 @@ public NodeDescriptor Create(string projectFullPath, IReadOnlyDictionary Date: Fri, 6 Sep 2024 10:44:59 -0600 Subject: [PATCH 10/18] Dump log in more places --- src/Common/MSBuildCachePluginBase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index d909e27..364585d 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -467,6 +467,7 @@ private async Task HandleProjectFinishedInnerAsync(FileAccessContext fileAccessC // If file access reports are disabled in MSBuild we can't cache anything as we don't know what to cache. if (!_hasHadFileAccessReport) { + await DumpFingerprintLogAsync(logger, nodeContext, null); return; } From 0be1a293fdf6e70eb44b9befd0d6b70568346951 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 6 Sep 2024 11:02:57 -0600 Subject: [PATCH 11/18] Ignore more inputs --- src/Common/Hashing/DirectoryFileHasher.cs | 7 +++++++ src/Common/build/Microsoft.MSBuildCache.Common.targets | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/src/Common/Hashing/DirectoryFileHasher.cs b/src/Common/Hashing/DirectoryFileHasher.cs index ae6557e..1be916c 100644 --- a/src/Common/Hashing/DirectoryFileHasher.cs +++ b/src/Common/Hashing/DirectoryFileHasher.cs @@ -46,6 +46,13 @@ public DirectoryFileHasher(string directory, IContentHasher contentHasher) return null; } +#if NETFRAMEWORK + if (path.IndexOf("microsoft.msbuildcache.", StringComparison.OrdinalIgnoreCase) >= 0) + { + return null; + } +#endif + ContentHash contentHash = await _contentHasher.GetFileHashAsync(path); return contentHash.ToHashByteArray(); })); diff --git a/src/Common/build/Microsoft.MSBuildCache.Common.targets b/src/Common/build/Microsoft.MSBuildCache.Common.targets index 92d3de1..5b90027 100644 --- a/src/Common/build/Microsoft.MSBuildCache.Common.targets +++ b/src/Common/build/Microsoft.MSBuildCache.Common.targets @@ -24,6 +24,11 @@ $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCachePackageVersion $(MSBuildCacheGlobalPropertiesToIgnore);IsGraphBuild $(MSBuildCacheGlobalPropertiesToIgnore);UseHostCompilerIfAvailable + $(MSBuildCacheGlobalPropertiesToIgnore);LangID + $(MSBuildCacheGlobalPropertiesToIgnore);LangName + $(MSBuildCacheGlobalPropertiesToIgnore);DefineExplicitDefaults + $(MSBuildCacheGlobalPropertiesToIgnore);EnableCaching + $(MSBuildCacheGlobalPropertiesToIgnore);VSIDEResolvedNonMSBuildProjectOutputs From 80e683f4802cdc6d431ab2a7aa6d9b93b7dd6eeb Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 6 Sep 2024 11:11:31 -0600 Subject: [PATCH 12/18] Ignore more --- src/Common/Fingerprinting/FingerprintFactory.cs | 2 +- src/Common/build/Microsoft.MSBuildCache.Common.targets | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/Fingerprinting/FingerprintFactory.cs b/src/Common/Fingerprinting/FingerprintFactory.cs index ef19cc8..28b4d3d 100644 --- a/src/Common/Fingerprinting/FingerprintFactory.cs +++ b/src/Common/Fingerprinting/FingerprintFactory.cs @@ -103,7 +103,7 @@ void AddSettingToFingerprint(IReadOnlyCollection? patterns, string setting string vcToolsVersion = nodeContext.Node.ProjectInstance.GetPropertyValue("VCToolsVersion"); if (!string.IsNullOrEmpty(vcToolsVersion)) { - entries.Add(CreateFingerprintEntry($"VCToolsVersion: {vcToolsVersion}")); + //entries.Add(CreateFingerprintEntry($"VCToolsVersion: {vcToolsVersion}")); } // If the .NET SDK changes, the node should rebuild. diff --git a/src/Common/build/Microsoft.MSBuildCache.Common.targets b/src/Common/build/Microsoft.MSBuildCache.Common.targets index 5b90027..b706e41 100644 --- a/src/Common/build/Microsoft.MSBuildCache.Common.targets +++ b/src/Common/build/Microsoft.MSBuildCache.Common.targets @@ -29,6 +29,7 @@ $(MSBuildCacheGlobalPropertiesToIgnore);DefineExplicitDefaults $(MSBuildCacheGlobalPropertiesToIgnore);EnableCaching $(MSBuildCacheGlobalPropertiesToIgnore);VSIDEResolvedNonMSBuildProjectOutputs + $(MSBuildCacheGlobalPropertiesToIgnore);DevEnvDir From dd0e3732cc4627d975331ffe0615c3b5023d382c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 6 Sep 2024 12:40:05 -0600 Subject: [PATCH 13/18] Release log file handle --- src/Common/Caching/CacheClient.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Common/Caching/CacheClient.cs b/src/Common/Caching/CacheClient.cs index c94ce22..dbfa99c 100644 --- a/src/Common/Caching/CacheClient.cs +++ b/src/Common/Caching/CacheClient.cs @@ -15,6 +15,7 @@ using BuildXL.Cache.ContentStore.Interfaces.Sessions; using BuildXL.Cache.ContentStore.Interfaces.Tracing; using BuildXL.Cache.ContentStore.Interfaces.Utils; +using BuildXL.Cache.ContentStore.Logging; using BuildXL.Cache.ContentStore.Tracing; using BuildXL.Cache.ContentStore.UtilitiesCore; using BuildXL.Cache.MemoizationStore.Interfaces.Caches; @@ -167,6 +168,13 @@ public virtual async ValueTask DisposeAsync() } _hasher.Dispose(); + + // HACK + foreach (ILog log in ((Logger)RootContext.Logger).GetLog()) + { + log.Dispose(); + } + RootContext.Logger.Dispose(); } From 0e064f8fbde98a3b468c52be550bada06ae639de Mon Sep 17 00:00:00 2001 From: David Federman Date: Mon, 14 Oct 2024 16:50:40 -0700 Subject: [PATCH 14/18] Add ability to query nodes recursively if needed --- src/Common.Tests/PluginSettingsTests.cs | 8 +- src/Common/MSBuildCachePluginBase.cs | 75 ++++++++++++++++++- src/Common/PluginSettings.cs | 2 + .../Microsoft.MSBuildCache.Common.targets | 2 + 4 files changed, 82 insertions(+), 5 deletions(-) diff --git a/src/Common.Tests/PluginSettingsTests.cs b/src/Common.Tests/PluginSettingsTests.cs index dc7c598..9ea0fef 100644 --- a/src/Common.Tests/PluginSettingsTests.cs +++ b/src/Common.Tests/PluginSettingsTests.cs @@ -351,16 +351,20 @@ public void GlobalPropertiesToIgnoreSetting(string? settingValue, string[] expec CollectionAssert.AreEqual(expectedValue, pluginSettings.GlobalPropertiesToIgnore.ToList()); } + [TestMethod] + public void GetResultsForUnqueriedDependenciesSetting() + => TestBoolSetting(nameof(PluginSettings.GetResultsForUnqueriedDependencies), pluginSettings => pluginSettings.GetResultsForUnqueriedDependencies); + private static void TestBoolSetting(string settingName, Func valueAccessor) => TestBasicSetting( settingName, valueAccessor, - testValues: new[] { false, true }); + testValues: [false, true]); private static void TestBasicSetting( string settingName, Func valueAccessor, - T[] testValues) + ReadOnlySpan testValues) { T defaultValue = valueAccessor(DefaultPluginSettings); diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index 7cb94e0..9511112 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -67,6 +67,7 @@ public abstract class MSBuildCachePluginBase : ProjectCachePlug private DirectoryLock? _localCacheDirectoryLock; private SemaphoreSlim? _singlePluginInstanceMutex; private PathNormalizer? _pathNormalizer; + private Func>? _getCacheResultAsync; private int _cacheHitCount; private long _cacheHitDurationMilliseconds; @@ -122,7 +123,8 @@ static MSBuildCachePluginBase() => nameof(_fileAccessRepository), nameof(_cacheClient), nameof(_ignoredOutputPatterns), - nameof(_identicalDuplicateOutputPatterns) + nameof(_identicalDuplicateOutputPatterns), + nameof(_getCacheResultAsync) )] protected bool Initialized { get; private set; } @@ -321,6 +323,16 @@ private async Task BeginBuildInnerAsync(CacheContext context, PluginLoggerBase l _fileAccessRepository = new FileAccessRepository(logger, Settings); _cacheClient = await CreateCacheClientAsync(logger, cancellationToken); + if (Settings.GetResultsForUnqueriedDependencies) + { + ConcurrentDictionary>> cacheResults = new(concurrencyLevel: Environment.ProcessorCount, _nodeContexts.Count); + _getCacheResultAsync = (nodeContext, logger, cancellationToken) => GetCacheResultRecursivelyAsync(cacheResults, nodeContext, logger, cancellationToken); + } + else + { + _getCacheResultAsync = GetCacheResultNonRecursiveAsync; + } + // Ensure all logs are written await Task.WhenAll(dumpParserInfoTasks); await dumpNodeContextsTask; @@ -372,14 +384,71 @@ private async Task GetCacheResultInnerAsync(BuildRequestData buildR return CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable); } - nodeContext.SetStartTime(); - if (!nodeContext.TargetNames.SetEquals(buildRequest.TargetNames)) { logger.LogMessage($"`TargetNames` does not match for {nodeContext.Id}. `{string.Join(";", nodeContext.TargetNames)}` vs `{string.Join(";", buildRequest.TargetNames)}`."); return CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable); } + return await _getCacheResultAsync(nodeContext, logger, cancellationToken); + } + + private async Task GetCacheResultRecursivelyAsync( + ConcurrentDictionary>> cacheResults, + NodeContext nodeContext, + PluginLoggerBase logger, + CancellationToken cancellationToken) + { + // Ensure we only query a node exactly once. MSBuild won't query a project more than once, but when recursion is enabled, + // we might have multiple build requests querying the same unqueried dependency. + return await cacheResults.GetOrAdd(nodeContext, new Lazy>( + async () => + { + foreach (NodeContext dependency in nodeContext.Dependencies) + { + if (dependency.BuildResult == null) + { + logger.LogMessage($"Querying cache for missing build result for dependency '{dependency.Id}'"); + CacheResult dependencyResult = await GetCacheResultRecursivelyAsync(cacheResults, dependency, logger, cancellationToken); + logger.LogMessage($"Dependency '{dependency.Id}' cache result: '{dependencyResult.ResultType}'"); + + if (dependencyResult.ResultType != CacheResultType.CacheHit) + { + logger.LogMessage($"Cache miss due to failed build result for dependency '{dependency.Id}'"); + Interlocked.Increment(ref _cacheMissCount); + return CacheResult.IndicateNonCacheHit(CacheResultType.CacheMiss); + } + } + } + + return await GetCacheResultSingleAsync(nodeContext, logger, cancellationToken); + })).Value; + } + + private async Task GetCacheResultNonRecursiveAsync(NodeContext nodeContext, PluginLoggerBase logger, CancellationToken cancellationToken) + { + foreach (NodeContext dependency in nodeContext.Dependencies) + { + if (dependency.BuildResult == null) + { + logger.LogMessage($"Cache miss due to failed or missing build result for dependency '{dependency.Id}'"); + Interlocked.Increment(ref _cacheMissCount); + return CacheResult.IndicateNonCacheHit(CacheResultType.CacheMiss); + } + } + + return await GetCacheResultSingleAsync(nodeContext, logger, cancellationToken); + } + + private async Task GetCacheResultSingleAsync(NodeContext nodeContext, PluginLoggerBase logger, CancellationToken cancellationToken) + { + if (!Initialized) + { + throw new InvalidOperationException(); + } + + nodeContext.SetStartTime(); + (PathSet? pathSet, NodeBuildResult? nodeBuildResult) = await _cacheClient.GetNodeAsync(nodeContext, cancellationToken); if (nodeBuildResult is null) { diff --git a/src/Common/PluginSettings.cs b/src/Common/PluginSettings.cs index fed4b7b..a10dd70 100644 --- a/src/Common/PluginSettings.cs +++ b/src/Common/PluginSettings.cs @@ -101,6 +101,8 @@ public string LocalCacheRootPath public IReadOnlyList GlobalPropertiesToIgnore { get; init; } = Array.Empty(); + public bool GetResultsForUnqueriedDependencies { get; init; } + public static T Create( IReadOnlyDictionary settings, PluginLoggerBase logger, diff --git a/src/Common/build/Microsoft.MSBuildCache.Common.targets b/src/Common/build/Microsoft.MSBuildCache.Common.targets index 75cc593..7457415 100644 --- a/src/Common/build/Microsoft.MSBuildCache.Common.targets +++ b/src/Common/build/Microsoft.MSBuildCache.Common.targets @@ -20,6 +20,7 @@ $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheAllowProcessCloseAfterProjectFinishProcessPatterns $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheGlobalPropertiesToIgnore + $(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheGetResultsForUnqueriedDependencies @@ -39,6 +40,7 @@ $(MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns) $(MSBuildCacheAllowProcessCloseAfterProjectFinishProcessPatterns) $(MSBuildCacheGlobalPropertiesToIgnore) + $(MSBuildCacheGetResultsForUnqueriedDependencies) From 8e9cc10f87145b200a4819a799e0fdcc7b6c5dbc Mon Sep 17 00:00:00 2001 From: Lifeng Lu Date: Wed, 16 Oct 2024 18:15:50 -0700 Subject: [PATCH 15/18] Recover a workaround. --- src/Common/MSBuildCachePluginBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index a835088..ff034a2 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -386,7 +386,7 @@ private async Task GetCacheResultInnerAsync(BuildRequestData buildR nodeContext.SetStartTime(); - if (!nodeContext.TargetNames.SetEquals(buildRequest.TargetNames)) + if (!nodeContext.TargetNames.IsSupersetOf(buildRequest.TargetNames)) { logger.LogMessage($"`TargetNames` does not match for {nodeContext.Id}. `{string.Join(";", nodeContext.TargetNames)}` vs `{string.Join(";", buildRequest.TargetNames)}`."); return CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable); From daf55c75e36cf9d5d294a01e263256707f2a671c Mon Sep 17 00:00:00 2001 From: Lifeng Lu Date: Fri, 18 Oct 2024 14:06:07 -0700 Subject: [PATCH 16/18] Revert unnecessary workaround. --- src/Common/NodeDescriptorFactory.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Common/NodeDescriptorFactory.cs b/src/Common/NodeDescriptorFactory.cs index 3743fed..57a25ac 100644 --- a/src/Common/NodeDescriptorFactory.cs +++ b/src/Common/NodeDescriptorFactory.cs @@ -38,11 +38,6 @@ public NodeDescriptor Create(string projectFullPath, IEnumerable Date: Mon, 28 Oct 2024 14:35:16 -0700 Subject: [PATCH 17/18] remove logic to switch .props/.targets for the plug-in itself. --- src/Common/Hashing/DirectoryFileHasher.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Common/Hashing/DirectoryFileHasher.cs b/src/Common/Hashing/DirectoryFileHasher.cs index 1be916c..ae6557e 100644 --- a/src/Common/Hashing/DirectoryFileHasher.cs +++ b/src/Common/Hashing/DirectoryFileHasher.cs @@ -46,13 +46,6 @@ public DirectoryFileHasher(string directory, IContentHasher contentHasher) return null; } -#if NETFRAMEWORK - if (path.IndexOf("microsoft.msbuildcache.", StringComparison.OrdinalIgnoreCase) >= 0) - { - return null; - } -#endif - ContentHash contentHash = await _contentHasher.GetFileHashAsync(path); return contentHash.ToHashByteArray(); })); From d1b72c39b38c74338c98b24ff5a3ee0bc288e73d Mon Sep 17 00:00:00 2001 From: Lifeng Lu Date: Wed, 30 Oct 2024 15:55:23 -0700 Subject: [PATCH 18/18] Improve some loggings --- src/Common/Caching/CacheClient.cs | 1 + src/Common/MSBuildCachePluginBase.cs | 72 +++++++++++++++++++--------- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/Common/Caching/CacheClient.cs b/src/Common/Caching/CacheClient.cs index 1d3fcb7..8d831a2 100644 --- a/src/Common/Caching/CacheClient.cs +++ b/src/Common/Caching/CacheClient.cs @@ -385,6 +385,7 @@ await AddNodeAsync( if (!selector.HasValue) { // GetMatchingSelectorAsync logs sufficiently + MSBuildCachePluginBase.DumpPartialFingerprintLog("weak", nodeContext, weakFingerprint); return (null, null); } diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index ff034a2..3bb4fe7 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -368,6 +368,7 @@ private async Task GetCacheResultInnerAsync(BuildRequestData buildR if (!Initialized) { // BeginBuild didn't finish successfully. It's expected to log sufficiently, so just bail. + logger.LogWarning($"Cache Miss for build {buildRequest.ProjectFullPath}: cache is not initialized."); return CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable); } @@ -381,6 +382,7 @@ private async Task GetCacheResultInnerAsync(BuildRequestData buildR NodeDescriptor nodeDescriptor = _nodeDescriptorFactory.Create(projectInstance); if (!_nodeContexts.TryGetValue(nodeDescriptor, out NodeContext? nodeContext)) { + logger.LogWarning($"Cache Miss for build {buildRequest.ProjectFullPath}: cannot find node context {nodeDescriptor}"); return CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable); } @@ -388,11 +390,16 @@ private async Task GetCacheResultInnerAsync(BuildRequestData buildR if (!nodeContext.TargetNames.IsSupersetOf(buildRequest.TargetNames)) { - logger.LogMessage($"`TargetNames` does not match for {nodeContext.Id}. `{string.Join(";", nodeContext.TargetNames)}` vs `{string.Join(";", buildRequest.TargetNames)}`."); + logger.LogWarning($"Cache Miss for build {buildRequest.ProjectFullPath}: `TargetNames` does not match for {nodeContext.Id}. `{string.Join(";", nodeContext.TargetNames)}` vs `{string.Join(";", buildRequest.TargetNames)}`."); return CacheResult.IndicateNonCacheHit(CacheResultType.CacheNotApplicable); } - return await _getCacheResultAsync(nodeContext, logger, cancellationToken); + var result = await _getCacheResultAsync(nodeContext, logger, cancellationToken); + + string targets = string.Join(";", buildRequest.TargetNames); + logger.LogWarning($"Cache {result.ResultType} for build {buildRequest.ProjectFullPath} ({targets})"); + + return result; } private async Task GetCacheResultRecursivelyAsync( @@ -416,7 +423,7 @@ private async Task GetCacheResultRecursivelyAsync( if (dependencyResult.ResultType != CacheResultType.CacheHit) { - logger.LogMessage($"Cache miss due to failed build result for dependency '{dependency.Id}'"); + logger.LogWarning($"Cache miss due to failed build result for dependency '{dependency.Id}'"); Interlocked.Increment(ref _cacheMissCount); return CacheResult.IndicateNonCacheHit(CacheResultType.CacheMiss); } @@ -433,7 +440,7 @@ private async Task GetCacheResultNonRecursiveAsync(NodeContext node { if (dependency.BuildResult == null) { - logger.LogMessage($"Cache miss due to failed or missing build result for dependency '{dependency.Id}'"); + logger.LogWarning($"Cache miss due to failed or missing build result for dependency '{dependency.Id}'"); Interlocked.Increment(ref _cacheMissCount); return CacheResult.IndicateNonCacheHit(CacheResultType.CacheMiss); } @@ -454,6 +461,8 @@ private async Task GetCacheResultSingleAsync(NodeContext nodeContex (PathSet? pathSet, NodeBuildResult? nodeBuildResult) = await _cacheClient.GetNodeAsync(nodeContext, cancellationToken); if (nodeBuildResult is null) { + logger.LogWarning($"Cache miss due to failed to find build result '{nodeContext.Id}'"); + Interlocked.Increment(ref _cacheMissCount); return CacheResult.IndicateNonCacheHit(CacheResultType.CacheMiss); } @@ -870,36 +879,53 @@ private async Task DumpFingerprintLogAsync( { logger.LogWarning($"Non-fatal exception while writing {filePath}. {ex.GetType().Name}: {ex.Message}"); } + } - static void WriteFingerprintJson(Utf8JsonWriter jsonWriter, string propertyName, Fingerprint? fingerprint) + internal static void DumpPartialFingerprintLog(string fingerprintType, NodeContext nodeContext, Fingerprint fingerprint) + { + string filePath = Path.Combine(nodeContext.LogDirectory, $"fingerprint_{fingerprintType}.json"); + try { - jsonWriter.WritePropertyName(propertyName); - - if (fingerprint == null) - { - jsonWriter.WriteNullValue(); - return; - } + using FileStream fileStream = File.Create(filePath); + using var jsonWriter = new Utf8JsonWriter(fileStream, SerializationHelper.WriterOptions); jsonWriter.WriteStartObject(); + WriteFingerprintJson(jsonWriter, fingerprintType, fingerprint); + jsonWriter.WriteEndObject(); + } + catch (Exception) + { + } + } - jsonWriter.WriteString("hash", fingerprint.Hash.ToHex()); + private static void WriteFingerprintJson(Utf8JsonWriter jsonWriter, string propertyName, Fingerprint? fingerprint) + { + jsonWriter.WritePropertyName(propertyName); - jsonWriter.WritePropertyName("entries"); - jsonWriter.WriteStartArray(); + if (fingerprint == null) + { + jsonWriter.WriteNullValue(); + return; + } - foreach (FingerprintEntry entry in fingerprint.Entries) - { - jsonWriter.WriteStartObject(); - jsonWriter.WriteString("hash", entry.Hash.ToHex()); - jsonWriter.WriteString("description", entry.Description); - jsonWriter.WriteEndObject(); - } + jsonWriter.WriteStartObject(); - jsonWriter.WriteEndArray(); // entries array + jsonWriter.WriteString("hash", fingerprint.Hash.ToHex()); + jsonWriter.WritePropertyName("entries"); + jsonWriter.WriteStartArray(); + + foreach (FingerprintEntry entry in fingerprint.Entries) + { + jsonWriter.WriteStartObject(); + jsonWriter.WriteString("hash", entry.Hash.ToHex()); + jsonWriter.WriteString("description", entry.Description); jsonWriter.WriteEndObject(); } + + jsonWriter.WriteEndArray(); // entries array + + jsonWriter.WriteEndObject(); } private static async Task DumpBuildResultLogAsync(