From b684493141c5a94183f6c2abe4dd3f60a1d78f5e Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Tue, 25 Nov 2025 09:00:16 +0100 Subject: [PATCH 01/13] Added a FIleHasProvider implementation for Unity Version Control (Plastic SCM) Using the cm ls command https://docs.unity.com/en-us/unity-version-control/uvcs-cli/list --- ...UnityVersionControlFileHashProviderTest.cs | 49 +++++++ src/Common/HexUtilities.cs | 6 + src/Common/MSBuildCachePluginBase.cs | 39 ++++-- .../SourceControl/GitFileHashProvider.cs | 2 +- .../UnityVersionControl.cs | 91 +++++++++++++ .../UnityVersionControlFileHashProvider.cs | 123 ++++++++++++++++++ 6 files changed, 297 insertions(+), 13 deletions(-) create mode 100644 src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs create mode 100644 src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs create mode 100644 src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs diff --git a/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs new file mode 100644 index 0000000..4b2deb1 --- /dev/null +++ b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using BuildXL.Utilities; +using Microsoft.MSBuildCache.SourceControl.UnityVersionControl; +using Microsoft.MSBuildCache.Tests.Mocks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.MSBuildCache.Tests.SourceControl; + +[TestClass] +public class UnityVersionControlFileHashProviderTests +{ + private const string RepoRoot = @"C:\work\MSBuildCacheTest"; + + private static readonly byte[] FakeHash = { 0, 1, 2, 3, 4 }; + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. Justification: Always set by MSTest + public TestContext TestContext { get; set; } +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + + private static Task FakeHasher(List filesToRehash, Dictionary fileHashes) + { + foreach (string file in filesToRehash) + { + fileHashes[file] = FakeHash; + } + + return Task.CompletedTask; + } + + [TestMethod] + public async Task ParseCmLsFiles() + { + // This has two modified and one untracked files + const string lsFilesOutput = "c:\\work\\MSBuildCacheTest\tFZMuOF2WDemh7irROkxyWw==\nc:\\work\\MSBuildCacheTest\\foo.txt\t9nwry/z6MPzLNvctyiKoFw==\nc:\\work\\MSBuildCacheTest\\bar.txt\t"; + + UnityVersionControlFileHashProvider unityFileHashProvider = new(NullPluginLogger.Instance); + Dictionary hashes = await unityFileHashProvider.ParseUnityLsFiles(new StringReader(lsFilesOutput), FakeHasher); + int filesExpected = 3; + Assert.AreEqual(filesExpected, hashes.Count, $"should be {filesExpected} files in this output"); + string barPath = Path.Combine(RepoRoot, @"bar.txt"); + Assert.AreEqual(FakeHash, hashes[barPath], $"bytes of {barPath} should be {FakeHash} since it should have gotten hashed by the FakeHasher"); + Assert.AreEqual("0001020304", hashes[Path.Combine(RepoRoot, "bar.txt")].ToHex()); + } +} \ No newline at end of file diff --git a/src/Common/HexUtilities.cs b/src/Common/HexUtilities.cs index 7959592..51411cc 100644 --- a/src/Common/HexUtilities.cs +++ b/src/Common/HexUtilities.cs @@ -64,4 +64,10 @@ public static byte[] HexToBytes(ReadOnlySpan hex) return result; } + + public static byte[] Base64ToBytes(string base64Hash) + { + byte[] hashBytes = Convert.FromBase64String(base64Hash); + return hashBytes; + } } diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index c9688f3..74a7b2a 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -33,6 +33,7 @@ using Microsoft.MSBuildCache.Hashing; using Microsoft.MSBuildCache.Parsing; using Microsoft.MSBuildCache.SourceControl; +using Microsoft.MSBuildCache.SourceControl.UnityVersionControl; namespace Microsoft.MSBuildCache; @@ -44,6 +45,7 @@ public abstract class MSBuildCachePluginBase : MSBuildCachePluginBase : ProjectCachePluginBase, IAsyncDisposable where TPluginSettings : PluginSettings { + enum VersionControl { git, uvcs }; private static readonly string PluginAssemblyDirectory = Path.GetDirectoryName(typeof(MSBuildCachePluginBase).Assembly.Location)!; private static readonly SemaphoreSlim SinglePluginInstanceLock = new(1, 1); @@ -53,6 +55,7 @@ public abstract class MSBuildCachePluginBase : ProjectCachePlug private string? _repoRoot; private string? _buildId; + private VersionControl? _versionControl; // Set if we've received any file access report. Ideally MSBuild would tell us in the CacheContext private bool _hasHadFileAccessReport; @@ -215,7 +218,7 @@ private async Task BeginBuildInnerAsync(CacheContext context, PluginLoggerBase l { _pluginLogger = logger; - _repoRoot = GetRepoRoot(context, logger); + (_repoRoot, _versionControl) = GetRepoRoot(context, logger); if (_repoRoot == null) { return; @@ -1033,8 +1036,9 @@ private bool TryAcquireLock(PluginSettings settings, PluginLoggerBase logger) return true; } - private static string? GetRepoRoot(CacheContext context, PluginLoggerBase logger) + private static (string?, VersionControl?) GetRepoRoot(CacheContext context, PluginLoggerBase logger) { + VersionControl versionControl = VersionControl.git; IEnumerable projectFilePaths = context.Graph != null ? context.Graph.EntryPointNodes.Select(node => node.ProjectInstance.FullPath) : context.GraphEntryPoints != null @@ -1044,42 +1048,53 @@ private bool TryAcquireLock(PluginSettings settings, PluginLoggerBase logger) HashSet repoRoots = new(StringComparer.OrdinalIgnoreCase); foreach (string projectFilePath in projectFilePaths) { - string? repoRoot = GetRepoRootInternal(Path.GetDirectoryName(projectFilePath)!); + (string? repoRoot, VersionControl? vc) = GetRepoRootInternal(Path.GetDirectoryName(projectFilePath)!, logger); // Tolerate projects which aren't under any git repo. if (repoRoot != null) { repoRoots.Add(repoRoot); } + + if (vc != null) + { + versionControl = vc.Value; + } } if (repoRoots.Count == 0) { logger.LogWarning("No projects are under git source control. Disabling the cache."); - return null; + return (null, null); } if (repoRoots.Count == 1) { string repoRoot = repoRoots.First(); logger.LogMessage($"Repo root: {repoRoot}"); - return repoRoot; + return (repoRoot, versionControl); } logger.LogWarning($"Graph contains projects from multiple git repositories. Disabling the cache. Repo roots: {string.Join(", ", repoRoots)}"); - return null; + return (null, null); - static string? GetRepoRootInternal(string path) + static (string?, VersionControl?) GetRepoRootInternal(string path, PluginLoggerBase logger) { // Note: When using git worktrees, .git may be a file instead of a directory. string gitPath = Path.Combine(path, ".git"); if (Directory.Exists(gitPath) || File.Exists(gitPath)) { - return path; + return (path, VersionControl.git); + } + + string unityVersionControlPath = Path.Combine(path, ".plastic"); + if (Directory.Exists(unityVersionControlPath)) + { + return (path, VersionControl.uvcs); } string? parentDir = Path.GetDirectoryName(path); - return parentDir != null ? GetRepoRootInternal(parentDir) : null; + return parentDir != null ? GetRepoRootInternal(parentDir, logger) : (null, null); } } @@ -1121,15 +1136,15 @@ void WarnIfCowNotSupportedBetweenRepoRootAndPath(string path, string pathDescrip private async Task> GetSourceControlFileHashesAsync(PluginLoggerBase logger, CancellationToken cancellationToken) { - if (_repoRoot == null) + if (_repoRoot == null || _versionControl == null) { throw new InvalidOperationException($"{nameof(_repoRoot)} was unexpectedly null"); } - logger.LogMessage("Source Control: Getting hashes"); + logger.LogMessage($"Source Control: Getting hashes from {_versionControl.Value}"); Stopwatch stopwatch = Stopwatch.StartNew(); - GitFileHashProvider hashProvider = new(logger); + ISourceControlFileHashProvider hashProvider = _versionControl == VersionControl.git ? new GitFileHashProvider(logger) : new UnityVersionControlFileHashProvider(logger); IReadOnlyDictionary fileHashes = await hashProvider.GetFileHashesAsync(_repoRoot, cancellationToken); logger.LogMessage($"Source Control: File hashes query took {stopwatch.ElapsedMilliseconds} ms"); diff --git a/src/Common/SourceControl/GitFileHashProvider.cs b/src/Common/SourceControl/GitFileHashProvider.cs index 1f6f641..de55c09 100644 --- a/src/Common/SourceControl/GitFileHashProvider.cs +++ b/src/Common/SourceControl/GitFileHashProvider.cs @@ -232,7 +232,7 @@ internal Task GitHashObjectAsync(string basePath, List filesToRehash, Di { if (exitCode != 0) { - throw new SourceControlHashException("git ls-files failed with exit code " + exitCode); + throw new SourceControlHashException("git hash-object failed with exit code " + exitCode); } return result; diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs new file mode 100644 index 0000000..db21792 --- /dev/null +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs @@ -0,0 +1,91 @@ + +using System; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Experimental.ProjectCache; + +namespace Microsoft.MSBuildCache.SourceControl.UnityVersionControl +{ + internal static class UnityVersionControl + { + public static async Task RunAsync( + PluginLoggerBase logger, + string workingDir, string args, + Func> onRunning, + Func onExit, + CancellationToken cancellationToken) + { + using Process process = new(); + process.StartInfo.FileName = "cm"; // Unity Version Control command line "cm" is expected to be on the PATH + process.StartInfo.Arguments = args; + process.StartInfo.UseShellExecute = false; + process.StartInfo.CreateNoWindow = true; + process.StartInfo.WorkingDirectory = workingDir; + process.StartInfo.RedirectStandardInput = true; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = Encoding.UTF8; + + Stopwatch sw = Stopwatch.StartNew(); + + process.Start(); + + static void KillProcess(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(); + } + } + catch + { + // Swallow. This is best-effort + } + } + + using (cancellationToken.Register(() => KillProcess(process))) + { + using (StreamWriter stdin = process.StandardInput) + using (StreamReader stdout = process.StandardOutput) + using (StreamReader stderr = process.StandardError) + { + + Task resultTask = Task.Run(async () => + { + try + { + return await onRunning(stdin, stdout); + } + finally + { + stdin.Close(); + } + }); +#if NETFRAMEWORK + process.WaitForExit(); + cancellationToken.ThrowIfCancellationRequested(); +#else + await process.WaitForExitAsync(cancellationToken); +#endif + + Task errorTask = Task.Run(() => stderr.ReadToEndAsync()); + if (process.ExitCode == 0) + { + logger.LogMessage($"cm.exe {args} (@{process.StartInfo.WorkingDirectory}) took {sw.ElapsedMilliseconds} msec and returned {process.ExitCode}."); + } + else + { + logger.LogMessage($"cm.exe {args} (@{process.StartInfo.WorkingDirectory}) took {sw.ElapsedMilliseconds} msec and returned {process.ExitCode}. Stderr: {await errorTask}"); + } + + return onExit(process.ExitCode, await resultTask); + } + } + } + } +} diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs new file mode 100644 index 0000000..91e780f --- /dev/null +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using BuildXL.Utilities.Core.Tasks; +using Microsoft.Build.Experimental.ProjectCache; + +namespace Microsoft.MSBuildCache.SourceControl.UnityVersionControl +{ + internal class UnityVersionControlFileHashProvider : ISourceControlFileHashProvider + { + private readonly PluginLoggerBase _logger; + public UnityVersionControlFileHashProvider(PluginLoggerBase logger) + { + _logger = logger; + } + public async Task> GetFileHashesAsync(string repoRoot, CancellationToken cancellationToken) + { + Task> hashesTask = GetRepoFileHashesAsync(repoRoot, cancellationToken); + return await hashesTask; + } + + private async Task> GetRepoFileHashesAsync(string basePath, CancellationToken cancellationToken) + { + return await UnityVersionControl.RunAsync(_logger, workingDir: basePath, "ls -R --format=\"{path}\t{hash}\"", + (_, stdout) => Task.Run(() => ParseUnityLsFiles(stdout, (filesToRehash, fileHashes) => GitHashObjectAsync(basePath, filesToRehash, fileHashes, cancellationToken))), + (exitCode, result) => + { + if (exitCode != 0) + { + throw new SourceControlHashException("cm ls failed with exit code " + exitCode); + } + + return result; + }, + cancellationToken); + } + + internal async Task> ParseUnityLsFiles( + TextReader cmOutput, + Func, Dictionary, Task> hasher) + { + // relativePathInRepositoryhash + //using var reader = new GitLsFileOutputReader(cmOutput); + var fileHashes = new Dictionary(StringComparer.OrdinalIgnoreCase); + var filesToRehash = new List(); + string? line; + while ((line = await cmOutput.ReadLineAsync()) != null) + { + var splitLine = line.ToString().Split('\t'); + string file = splitLine[0]; + if (splitLine.Length > 1 && splitLine[1].Length > 0) + { + string hash = splitLine[1]; + fileHashes[file] = HexUtilities.Base64ToBytes(hash); + } + else + { + filesToRehash.Add(file); + } + } + + if (filesToRehash.Count > 0) + { + Stopwatch sw = Stopwatch.StartNew(); + // we could do this as new files come in just not clear it's worth it + await hasher(filesToRehash, fileHashes); + _logger.LogMessage($"{fileHashes.Count} files Rehashing {filesToRehash.Count} modified files took {sw.ElapsedMilliseconds} msec"); + } + + return fileHashes; + } + + internal Task GitHashObjectAsync(string basePath, List filesToRehash, Dictionary filehashes, CancellationToken cancellationToken) + { + return Git.RunAsync( + _logger, + workingDir: basePath, + "hash-object --stdin-paths", + async (stdin, stdout) => + { + foreach (string file in filesToRehash) + { + string? gitHashOfFile; + + if (File.Exists(file)) + { + await stdin.WriteLineAsync(file); + gitHashOfFile = await stdout.ReadLineAsync(); + + if (string.IsNullOrWhiteSpace(gitHashOfFile)) + { + _logger.LogMessage($"git hash-object returned an empty string for {file}. Forcing a cache miss by using a Guid"); + + // Guids are only 32 characters and git hashes are 40. Prepend 8 characters to match and to generally be recognizable. + gitHashOfFile = "bad00000" + Guid.NewGuid().ToString("N"); + } + } + else + { + gitHashOfFile = null; + } + + filehashes[file] = HexUtilities.HexToBytes(gitHashOfFile); + } + + return Unit.Void; + }, + (exitCode, result) => + { + if (exitCode != 0) + { + throw new SourceControlHashException("git hash-object failed with exit code " + exitCode); + } + + return result; + }, + cancellationToken); + } + } +} From 2db7b62a7c862c887a6375260b9ca1ce7cf4a96e Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Thu, 27 Nov 2025 11:21:45 +0100 Subject: [PATCH 02/13] Added note about how solution and projects need to be structured for caching to work --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 538f4d9..d3e7633 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Here is an example if you're using NuGet's [Central Package Management](https:// ``` -For repos using C++, you will need to add the projects to a packages.config and import the props/targets files directly. +For repos using C++, you will need to add the projects to a packages.config and import the props/targets files directly. For caching to work, the target solution needs to be above its projects. `Directory.Build.props`: ```xml From 23ab1fbc3e2b76769a41efe1bdec2f8455f98741 Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Thu, 27 Nov 2025 11:27:20 +0100 Subject: [PATCH 03/13] Fixed the reading of the cm ls output when there's a lot of files --- .../UnityVersionControlFileHashProvider.cs | 79 ++++++++++++++++++- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs index 91e780f..5f24d4f 100644 --- a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; using BuildXL.Utilities.Core.Tasks; @@ -43,11 +45,11 @@ internal async Task> ParseUnityLsFiles( Func, Dictionary, Task> hasher) { // relativePathInRepositoryhash - //using var reader = new GitLsFileOutputReader(cmOutput); + using var reader = new UnityVersionContorlLsFileOutputReader(cmOutput); var fileHashes = new Dictionary(StringComparer.OrdinalIgnoreCase); var filesToRehash = new List(); - string? line; - while ((line = await cmOutput.ReadLineAsync()) != null) + StringBuilder? line; + while ((line = reader.ReadLine()) != null) { var splitLine = line.ToString().Split('\t'); string file = splitLine[0]; @@ -58,6 +60,7 @@ internal async Task> ParseUnityLsFiles( } else { + _logger.LogMessage($"{file} is missing a hash and will be rehashed."); filesToRehash.Add(file); } } @@ -119,5 +122,75 @@ internal Task GitHashObjectAsync(string basePath, List filesToRehash, Di }, cancellationToken); } + + private sealed class UnityVersionContorlLsFileOutputReader : IDisposable + { + readonly BlockingCollection _lines = new BlockingCollection(); + + public UnityVersionContorlLsFileOutputReader(TextReader reader) + { + Task.Run(() => PopulateAsync(reader)); + } + + private void PopulateAsync(TextReader reader) + { + int overflowLength = 0; + var buffer = new char[4096]; // must be large enough to hold at least one line of output + while (true) + { + int readCnt = reader.Read(buffer, overflowLength, buffer.Length - overflowLength); + if (readCnt == 0) // end of stream + { + if (overflowLength > 0) + { + _lines.Add(new StringBuilder(overflowLength).Append(buffer, 0, overflowLength)); + } + _lines.CompleteAdding(); + return; + } + + readCnt += overflowLength; + int startIdx = 0, eolIdx; + while (startIdx < readCnt && (eolIdx = Array.IndexOf(buffer, '\n', startIdx)) != -1) + { + int lineLength = eolIdx - startIdx; + if (overflowLength > 0) + { + overflowLength = 0; + startIdx = 0; + } + _lines.Add(new StringBuilder(lineLength).Append(buffer, startIdx, lineLength)); + startIdx = eolIdx + 1; + } + if (startIdx < readCnt) + { + if (overflowLength > 0) // we already have some overflow left, but the line could not fit the buffer + { + throw new InvalidDataException($"Internal: cm ls output line length {readCnt - startIdx} exceeds {nameof(buffer)} size {buffer.Length}. Increase the latter."); + } + overflowLength = readCnt - startIdx; + Array.Copy(buffer, startIdx, buffer, 0, overflowLength); + } + } + } + + public StringBuilder? ReadLine() + { + while (!_lines.IsCompleted) + { + if (_lines.TryTake(out StringBuilder? result, -1)) + { + return result; + } + } + return null; + } + + public void Dispose() + { + _lines.Dispose(); + } + } } + } From 53b463bad6eb4139a81e71da7a0fcee1b5fef231 Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Thu, 27 Nov 2025 11:27:49 +0100 Subject: [PATCH 04/13] Added test ParseRealCmLsFiles that is mainly useful during development --- .../UnityVersionControlFileHashProviderTest.cs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs index 4b2deb1..bce0003 100644 --- a/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs +++ b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; +using System.Threading; using System.Threading.Tasks; using BuildXL.Utilities; using Microsoft.MSBuildCache.SourceControl.UnityVersionControl; @@ -14,8 +15,6 @@ namespace Microsoft.MSBuildCache.Tests.SourceControl; [TestClass] public class UnityVersionControlFileHashProviderTests { - private const string RepoRoot = @"C:\work\MSBuildCacheTest"; - private static readonly byte[] FakeHash = { 0, 1, 2, 3, 4 }; #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. Justification: Always set by MSTest @@ -35,6 +34,7 @@ private static Task FakeHasher(List filesToRehash, Dictionary hashes = await unityFileHashProvider.ParseUnityLsFiles(new StringReader(lsFilesOutput), FakeHasher); int filesExpected = 3; Assert.AreEqual(filesExpected, hashes.Count, $"should be {filesExpected} files in this output"); - string barPath = Path.Combine(RepoRoot, @"bar.txt"); + string barPath = Path.Combine(repoRoot, @"bar.txt").ToUpperInvariant(); Assert.AreEqual(FakeHash, hashes[barPath], $"bytes of {barPath} should be {FakeHash} since it should have gotten hashed by the FakeHasher"); - Assert.AreEqual("0001020304", hashes[Path.Combine(RepoRoot, "bar.txt")].ToHex()); + Assert.AreEqual("0001020304", hashes[Path.Combine(repoRoot, "bar.txt")].ToHex()); + } + + [TestMethod, Ignore("Only useful if there is an UVCS workspace at the repoRoot")] + public async Task ParseRealCmLsFiles() + { + const string repoRoot = @"C:\work\MSBuildCacheTest"; + UnityVersionControlFileHashProvider unityFileHashProvider = new(NullPluginLogger.Instance); + var dict = await unityFileHashProvider.GetFileHashesAsync(repoRoot, CancellationToken.None); + int filesExpected = 116898; + Assert.AreEqual(filesExpected, dict.Count, $"should be {filesExpected} files in this output"); } } \ No newline at end of file From 103191f68157bb67fe60391a9ef7f13d0185da8c Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Thu, 27 Nov 2025 11:29:41 +0100 Subject: [PATCH 05/13] Updated README to mention that it requires Git or UVCS --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d3e7633..86cd542 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This project provides plugin implementations for the experimental [MSBuild Proje ## Usage -This feature requires Visual Studio 17.9 or later. +This feature requires Visual Studio 17.9 or later and assumes the repo uses Git or Unity Version Control. To enable caching, simply add a `` for the desired cache implementation and set various properties to configure it. From 417f756e6f94a082dadd390d1760d1d958698532 Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Fri, 28 Nov 2025 12:04:20 +0100 Subject: [PATCH 06/13] Added timeout to tests --- .../UnityVersionControlFileHashProviderTest.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs index bce0003..0acc667 100644 --- a/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs +++ b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs @@ -31,7 +31,7 @@ private static Task FakeHasher(List filesToRehash, Dictionary Date: Fri, 28 Nov 2025 12:10:51 +0100 Subject: [PATCH 07/13] Made reading in the output synchronous since it would otherwise was cause intermittent errors --- .../UnityVersionControlFileHashProviderTest.cs | 4 ++-- .../UnityVersionControl/UnityVersionControl.cs | 15 ++++----------- .../UnityVersionControlFileHashProvider.cs | 10 ++++++++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs index 0acc667..6301fbc 100644 --- a/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs +++ b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs @@ -47,10 +47,10 @@ public async Task ParseCmLsFiles() Assert.AreEqual("0001020304", hashes[Path.Combine(repoRoot, "bar.txt")].ToHex()); } - [TestMethod, Ignore, Timeout(10000)] + [TestMethod, Timeout(10000)] public async Task ParseRealCmLsFiles() { - const string repoRoot = @"C:\work\MSBuildCacheTestLarge"; + const string repoRoot = @"C:\work\devroot"; UnityVersionControlFileHashProvider unityFileHashProvider = new(NullPluginLogger.Instance); var dict = await unityFileHashProvider.GetFileHashesAsync(repoRoot, CancellationToken.None); int filesExpected = 116921; diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs index db21792..c1d113d 100644 --- a/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs @@ -14,7 +14,7 @@ internal static class UnityVersionControl public static async Task RunAsync( PluginLoggerBase logger, string workingDir, string args, - Func> onRunning, + Func> onRunning, Func onExit, CancellationToken cancellationToken) { @@ -54,18 +54,12 @@ static void KillProcess(Process process) using (StreamReader stdout = process.StandardOutput) using (StreamReader stderr = process.StandardError) { - Task resultTask = Task.Run(async () => { - try - { - return await onRunning(stdin, stdout); - } - finally - { - stdin.Close(); - } + return await onRunning(stdout); }); + Task errorTask = Task.Run(() => stderr.ReadToEndAsync()); + #if NETFRAMEWORK process.WaitForExit(); cancellationToken.ThrowIfCancellationRequested(); @@ -73,7 +67,6 @@ static void KillProcess(Process process) await process.WaitForExitAsync(cancellationToken); #endif - Task errorTask = Task.Run(() => stderr.ReadToEndAsync()); if (process.ExitCode == 0) { logger.LogMessage($"cm.exe {args} (@{process.StartInfo.WorkingDirectory}) took {sw.ElapsedMilliseconds} msec and returned {process.ExitCode}."); diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs index 5f24d4f..2a19e9e 100644 --- a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs @@ -27,7 +27,7 @@ public async Task> GetFileHashesAsync(string private async Task> GetRepoFileHashesAsync(string basePath, CancellationToken cancellationToken) { return await UnityVersionControl.RunAsync(_logger, workingDir: basePath, "ls -R --format=\"{path}\t{hash}\"", - (_, stdout) => Task.Run(() => ParseUnityLsFiles(stdout, (filesToRehash, fileHashes) => GitHashObjectAsync(basePath, filesToRehash, fileHashes, cancellationToken))), + async (stdout) => await ParseUnityLsFiles(stdout, (filesToRehash, fileHashes) => GitHashObjectAsync(basePath, filesToRehash, fileHashes, cancellationToken)), (exitCode, result) => { if (exitCode != 0) @@ -45,7 +45,9 @@ internal async Task> ParseUnityLsFiles( Func, Dictionary, Task> hasher) { // relativePathInRepositoryhash + _logger.LogMessage("Begin reading in the output"); using var reader = new UnityVersionContorlLsFileOutputReader(cmOutput); + _logger.LogMessage("Begin parsing the read output"); var fileHashes = new Dictionary(StringComparer.OrdinalIgnoreCase); var filesToRehash = new List(); StringBuilder? line; @@ -56,6 +58,10 @@ internal async Task> ParseUnityLsFiles( if (splitLine.Length > 1 && splitLine[1].Length > 0) { string hash = splitLine[1]; + if (hash.Length == 0) + { + throw new InvalidOperationException("hash cant be of zero length"); + } fileHashes[file] = HexUtilities.Base64ToBytes(hash); } else @@ -129,7 +135,7 @@ private sealed class UnityVersionContorlLsFileOutputReader : IDisposable public UnityVersionContorlLsFileOutputReader(TextReader reader) { - Task.Run(() => PopulateAsync(reader)); + PopulateAsync(reader); } private void PopulateAsync(TextReader reader) From d46229ad96dde62fc91028969dbde7a7e7cdfa54 Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Tue, 2 Dec 2025 13:18:19 +0100 Subject: [PATCH 08/13] Fix indentation --- .../UnityVersionControl/UnityVersionControl.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs index c1d113d..bd91abf 100644 --- a/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs @@ -12,11 +12,11 @@ namespace Microsoft.MSBuildCache.SourceControl.UnityVersionControl internal static class UnityVersionControl { public static async Task RunAsync( - PluginLoggerBase logger, - string workingDir, string args, - Func> onRunning, - Func onExit, - CancellationToken cancellationToken) + PluginLoggerBase logger, + string workingDir, string args, + Func> onRunning, + Func onExit, + CancellationToken cancellationToken) { using Process process = new(); process.StartInfo.FileName = "cm"; // Unity Version Control command line "cm" is expected to be on the PATH From 27fc8d81817861784ad9ea2551ff20936c6ae3f3 Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Tue, 2 Dec 2025 13:20:10 +0100 Subject: [PATCH 09/13] Optimized away an array allocation --- .../UnityVersionControlFileHashProvider.cs | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs index 2a19e9e..d47dcc8 100644 --- a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs @@ -45,24 +45,43 @@ internal async Task> ParseUnityLsFiles( Func, Dictionary, Task> hasher) { // relativePathInRepositoryhash - _logger.LogMessage("Begin reading in the output"); using var reader = new UnityVersionContorlLsFileOutputReader(cmOutput); - _logger.LogMessage("Begin parsing the read output"); var fileHashes = new Dictionary(StringComparer.OrdinalIgnoreCase); var filesToRehash = new List(); - StringBuilder? line; - while ((line = reader.ReadLine()) != null) + StringBuilder? lineSb; + while ((lineSb = reader.ReadLine()) != null) { - var splitLine = line.ToString().Split('\t'); - string file = splitLine[0]; - if (splitLine.Length > 1 && splitLine[1].Length > 0) + int delimiterIndex = -1; + char delimiter = '\t'; + for (int i = 0; i < lineSb.Length; i++) { - string hash = splitLine[1]; - if (hash.Length == 0) + if (lineSb[i] == delimiter) { - throw new InvalidOperationException("hash cant be of zero length"); + delimiterIndex = i; + break; + } + } + if (delimiterIndex == -1) + { + throw new InvalidDataException("Failed to split the string, missing a tab"); + } + + string file = lineSb.ToString(0, delimiterIndex); + int hashStartIndex = delimiterIndex + 1; + // Check that the line contains a hash, i.e. more than just the file path + if (hashStartIndex <= lineSb.Length) + { + string hash = lineSb.ToString(hashStartIndex, lineSb.Length - hashStartIndex); + + try + { + fileHashes[file] = Convert.FromBase64String(hash); + } + catch (FormatException fe) + { + // Add the file and hash to the exception so the log is a bit easier to troubleshoot + throw new FormatException($"Failed to convert base64 hash to bytes for file: {file}, hash: {hash}", fe); } - fileHashes[file] = HexUtilities.Base64ToBytes(hash); } else { From 0f1fc805a8339ad136daf032f00bb4bb33de9ec7 Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Tue, 2 Dec 2025 13:20:45 +0100 Subject: [PATCH 10/13] Fix feedback in MSBuildCachePluginBase --- src/Common/HexUtilities.cs | 6 --- src/Common/MSBuildCachePluginBase.cs | 59 +++++++++++++++++----------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/Common/HexUtilities.cs b/src/Common/HexUtilities.cs index 51411cc..7959592 100644 --- a/src/Common/HexUtilities.cs +++ b/src/Common/HexUtilities.cs @@ -64,10 +64,4 @@ public static byte[] HexToBytes(ReadOnlySpan hex) return result; } - - public static byte[] Base64ToBytes(string base64Hash) - { - byte[] hashBytes = Convert.FromBase64String(base64Hash); - return hashBytes; - } } diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index 74a7b2a..43d1852 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -45,7 +45,18 @@ public abstract class MSBuildCachePluginBase : MSBuildCachePluginBase : ProjectCachePluginBase, IAsyncDisposable where TPluginSettings : PluginSettings { - enum VersionControl { git, uvcs }; + private class RepoInfo + { + public string Root { get; set; } + public VersionControl VersionControl { get; set; } + + public RepoInfo(string root, VersionControl versionControl) + { + Root = root; + VersionControl = versionControl; + } + } + enum VersionControl { Git, UVCS }; private static readonly string PluginAssemblyDirectory = Path.GetDirectoryName(typeof(MSBuildCachePluginBase).Assembly.Location)!; private static readonly SemaphoreSlim SinglePluginInstanceLock = new(1, 1); @@ -217,12 +228,13 @@ public override Task BeginBuildAsync(CacheContext context, PluginLoggerBase logg private async Task BeginBuildInnerAsync(CacheContext context, PluginLoggerBase logger, CancellationToken cancellationToken) { _pluginLogger = logger; - - (_repoRoot, _versionControl) = GetRepoRoot(context, logger); - if (_repoRoot == null) + RepoInfo? repoInfo = GetRepoInfo(context, logger); + if (repoInfo == null) { return; } + _repoRoot = repoInfo.Root; + _versionControl = repoInfo.VersionControl; _buildId = GetBuildId(); @@ -1036,9 +1048,9 @@ private bool TryAcquireLock(PluginSettings settings, PluginLoggerBase logger) return true; } - private static (string?, VersionControl?) GetRepoRoot(CacheContext context, PluginLoggerBase logger) + private static RepoInfo? GetRepoInfo(CacheContext context, PluginLoggerBase logger) { - VersionControl versionControl = VersionControl.git; + RepoInfo? repoInfo = null; IEnumerable projectFilePaths = context.Graph != null ? context.Graph.EntryPointNodes.Select(node => node.ProjectInstance.FullPath) : context.GraphEntryPoints != null @@ -1048,53 +1060,46 @@ private static (string?, VersionControl?) GetRepoRoot(CacheContext context, Plug HashSet repoRoots = new(StringComparer.OrdinalIgnoreCase); foreach (string projectFilePath in projectFilePaths) { - (string? repoRoot, VersionControl? vc) = GetRepoRootInternal(Path.GetDirectoryName(projectFilePath)!, logger); - - // Tolerate projects which aren't under any git repo. - if (repoRoot != null) + repoInfo = GetRepoInfoInternal(Path.GetDirectoryName(projectFilePath)!, logger); + if (repoInfo != null) { - repoRoots.Add(repoRoot); - } - - if (vc != null) - { - versionControl = vc.Value; + repoRoots.Add(repoInfo.Root); } } if (repoRoots.Count == 0) { logger.LogWarning("No projects are under git source control. Disabling the cache."); - return (null, null); + return null; } if (repoRoots.Count == 1) { string repoRoot = repoRoots.First(); logger.LogMessage($"Repo root: {repoRoot}"); - return (repoRoot, versionControl); + return repoInfo; } logger.LogWarning($"Graph contains projects from multiple git repositories. Disabling the cache. Repo roots: {string.Join(", ", repoRoots)}"); - return (null, null); + return null; - static (string?, VersionControl?) GetRepoRootInternal(string path, PluginLoggerBase logger) + static RepoInfo? GetRepoInfoInternal(string path, PluginLoggerBase logger) { // Note: When using git worktrees, .git may be a file instead of a directory. string gitPath = Path.Combine(path, ".git"); if (Directory.Exists(gitPath) || File.Exists(gitPath)) { - return (path, VersionControl.git); + return new RepoInfo(path, VersionControl.Git); } string unityVersionControlPath = Path.Combine(path, ".plastic"); if (Directory.Exists(unityVersionControlPath)) { - return (path, VersionControl.uvcs); + return new RepoInfo(path, VersionControl.UVCS); } string? parentDir = Path.GetDirectoryName(path); - return parentDir != null ? GetRepoRootInternal(parentDir, logger) : (null, null); + return parentDir != null ? GetRepoInfoInternal(parentDir, logger) : null; } } @@ -1144,7 +1149,13 @@ private async Task> GetSourceControlFileHash logger.LogMessage($"Source Control: Getting hashes from {_versionControl.Value}"); Stopwatch stopwatch = Stopwatch.StartNew(); - ISourceControlFileHashProvider hashProvider = _versionControl == VersionControl.git ? new GitFileHashProvider(logger) : new UnityVersionControlFileHashProvider(logger); + ISourceControlFileHashProvider hashProvider = _versionControl switch + { + VersionControl.Git => new GitFileHashProvider(logger), + VersionControl.UVCS => new UnityVersionControlFileHashProvider(logger), + _ => throw new NotImplementedException() + }; + IReadOnlyDictionary fileHashes = await hashProvider.GetFileHashesAsync(_repoRoot, cancellationToken); logger.LogMessage($"Source Control: File hashes query took {stopwatch.ElapsedMilliseconds} ms"); From c86c46f3b52fb07fd0ae699ee4c19a75c1fe1ed0 Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Tue, 2 Dec 2025 13:30:26 +0100 Subject: [PATCH 11/13] Fixed logic error when checking if the file has a hash --- .../UnityVersionControl/UnityVersionControlFileHashProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs index d47dcc8..8f896d6 100644 --- a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs @@ -69,7 +69,7 @@ internal async Task> ParseUnityLsFiles( string file = lineSb.ToString(0, delimiterIndex); int hashStartIndex = delimiterIndex + 1; // Check that the line contains a hash, i.e. more than just the file path - if (hashStartIndex <= lineSb.Length) + if (hashStartIndex < lineSb.Length) { string hash = lineSb.ToString(hashStartIndex, lineSb.Length - hashStartIndex); From 8d214d3f4448ffb4eac33862fad384be222c11c6 Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Tue, 2 Dec 2025 13:49:03 +0100 Subject: [PATCH 12/13] Extracted out GitHashObjectAsync to Git.HashObjectAsync --- .../SourceControl/GitFileHashProviderTest.cs | Bin 13696 -> 13704 bytes src/Common/SourceControl/Git.cs | 49 +++++++++++++++++ .../SourceControl/GitFileHashProvider.cs | 49 +---------------- .../UnityVersionControlFileHashProvider.cs | 50 +----------------- 4 files changed, 51 insertions(+), 97 deletions(-) diff --git a/src/Common.Tests/SourceControl/GitFileHashProviderTest.cs b/src/Common.Tests/SourceControl/GitFileHashProviderTest.cs index 09e3955d26e1137e04c0694acfbdadd270168ed0..aa993819f27a69683772d68d0336f99aae83a98a 100644 GIT binary patch delta 25 gcmZq3?#SNoN14SvvqW#Qu!`a2Kx4_xPAYbS0Dz(hmjD0& delta 23 ecmeCkZphy7M|pCPp$M~kX2~QK!_7V_Hi7_h-3V3y diff --git a/src/Common/SourceControl/Git.cs b/src/Common/SourceControl/Git.cs index 7f94d43..3150755 100644 --- a/src/Common/SourceControl/Git.cs +++ b/src/Common/SourceControl/Git.cs @@ -1,10 +1,12 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; +using BuildXL.Utilities.Core.Tasks; using Microsoft.Build.Experimental.ProjectCache; #if NETFRAMEWORK using Process = Microsoft.MSBuildCache.SourceControl.GitProcess; @@ -95,4 +97,51 @@ static void KillProcess(Process process) } } } + + internal static Task HashObjectAsync(string basePath, List filesToRehash, Dictionary filehashes, PluginLoggerBase _logger, CancellationToken cancellationToken) + { + return RunAsync( + _logger, + workingDir: basePath, + "hash-object --stdin-paths", + async (stdin, stdout) => + { + foreach (string file in filesToRehash) + { + string? gitHashOfFile; + + if (File.Exists(file)) + { + await stdin.WriteLineAsync(file); + gitHashOfFile = await stdout.ReadLineAsync(); + + if (string.IsNullOrWhiteSpace(gitHashOfFile)) + { + _logger.LogMessage($"git hash-object returned an empty string for {file}. Forcing a cache miss by using a Guid"); + + // Guids are only 32 characters and git hashes are 40. Prepend 8 characters to match and to generally be recognizable. + gitHashOfFile = "bad00000" + Guid.NewGuid().ToString("N"); + } + } + else + { + gitHashOfFile = null; + } + + filehashes[file] = HexUtilities.HexToBytes(gitHashOfFile); + } + + return Unit.Void; + }, + (exitCode, result) => + { + if (exitCode != 0) + { + throw new SourceControlHashException("git hash-object failed with exit code " + exitCode); + } + + return result; + }, + cancellationToken); + } } \ No newline at end of file diff --git a/src/Common/SourceControl/GitFileHashProvider.cs b/src/Common/SourceControl/GitFileHashProvider.cs index de55c09..11d30fa 100644 --- a/src/Common/SourceControl/GitFileHashProvider.cs +++ b/src/Common/SourceControl/GitFileHashProvider.cs @@ -102,7 +102,7 @@ private async Task> GetModuleFileHashesAsync(string b _logger, workingDir: basePath, "ls-files -z -cmos --exclude-standard", - (_, stdout) => Task.Run(() => ParseGitLsFiles(basePath, stdout, (filesToRehash, fileHashes) => GitHashObjectAsync(basePath, filesToRehash, fileHashes, cancellationToken))), + (_, stdout) => Task.Run(() => ParseGitLsFiles(basePath, stdout, (filesToRehash, fileHashes) => Git.HashObjectAsync(basePath, filesToRehash, fileHashes, _logger, cancellationToken))), (exitCode, result) => { if (exitCode != 0) @@ -193,53 +193,6 @@ internal async Task> ParseGitLsFiles( return fileHashes; } - internal Task GitHashObjectAsync(string basePath, List filesToRehash, Dictionary filehashes, CancellationToken cancellationToken) - { - return Git.RunAsync( - _logger, - workingDir: basePath, - "hash-object --stdin-paths", - async (stdin, stdout) => - { - foreach (string file in filesToRehash) - { - string? gitHashOfFile; - - if (File.Exists(file)) - { - await stdin.WriteLineAsync(file); - gitHashOfFile = await stdout.ReadLineAsync(); - - if (string.IsNullOrWhiteSpace(gitHashOfFile)) - { - _logger.LogMessage($"git hash-object returned an empty string for {file}. Forcing a cache miss by using a Guid"); - - // Guids are only 32 characters and git hashes are 40. Prepend 8 characters to match and to generally be recognizable. - gitHashOfFile = "bad00000" + Guid.NewGuid().ToString("N"); - } - } - else - { - gitHashOfFile = null; - } - - filehashes[file] = HexUtilities.HexToBytes(gitHashOfFile); - } - - return Unit.Void; - }, - (exitCode, result) => - { - if (exitCode != 0) - { - throw new SourceControlHashException("git hash-object failed with exit code " + exitCode); - } - - return result; - }, - cancellationToken); - } - private Task> GetInitializedSubmodulesAsync(string repoRoot, CancellationToken cancellationToken) { string stdErrString = string.Empty; diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs index 8f896d6..832dde5 100644 --- a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs @@ -6,7 +6,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using BuildXL.Utilities.Core.Tasks; using Microsoft.Build.Experimental.ProjectCache; namespace Microsoft.MSBuildCache.SourceControl.UnityVersionControl @@ -27,7 +26,7 @@ public async Task> GetFileHashesAsync(string private async Task> GetRepoFileHashesAsync(string basePath, CancellationToken cancellationToken) { return await UnityVersionControl.RunAsync(_logger, workingDir: basePath, "ls -R --format=\"{path}\t{hash}\"", - async (stdout) => await ParseUnityLsFiles(stdout, (filesToRehash, fileHashes) => GitHashObjectAsync(basePath, filesToRehash, fileHashes, cancellationToken)), + async (stdout) => await ParseUnityLsFiles(stdout, (filesToRehash, fileHashes) => Git.HashObjectAsync(basePath, filesToRehash, fileHashes, _logger, cancellationToken)), (exitCode, result) => { if (exitCode != 0) @@ -101,53 +100,6 @@ internal async Task> ParseUnityLsFiles( return fileHashes; } - internal Task GitHashObjectAsync(string basePath, List filesToRehash, Dictionary filehashes, CancellationToken cancellationToken) - { - return Git.RunAsync( - _logger, - workingDir: basePath, - "hash-object --stdin-paths", - async (stdin, stdout) => - { - foreach (string file in filesToRehash) - { - string? gitHashOfFile; - - if (File.Exists(file)) - { - await stdin.WriteLineAsync(file); - gitHashOfFile = await stdout.ReadLineAsync(); - - if (string.IsNullOrWhiteSpace(gitHashOfFile)) - { - _logger.LogMessage($"git hash-object returned an empty string for {file}. Forcing a cache miss by using a Guid"); - - // Guids are only 32 characters and git hashes are 40. Prepend 8 characters to match and to generally be recognizable. - gitHashOfFile = "bad00000" + Guid.NewGuid().ToString("N"); - } - } - else - { - gitHashOfFile = null; - } - - filehashes[file] = HexUtilities.HexToBytes(gitHashOfFile); - } - - return Unit.Void; - }, - (exitCode, result) => - { - if (exitCode != 0) - { - throw new SourceControlHashException("git hash-object failed with exit code " + exitCode); - } - - return result; - }, - cancellationToken); - } - private sealed class UnityVersionContorlLsFileOutputReader : IDisposable { readonly BlockingCollection _lines = new BlockingCollection(); From a91ac93fa9bb15f3a3423ab4dfb746090258be0a Mon Sep 17 00:00:00 2001 From: eriknilsontenstar Date: Tue, 2 Dec 2025 13:51:08 +0100 Subject: [PATCH 13/13] Renamed function --- .../UnityVersionControlFileHashProvider.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs index 832dde5..9f9140d 100644 --- a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs @@ -106,10 +106,10 @@ private sealed class UnityVersionContorlLsFileOutputReader : IDisposable public UnityVersionContorlLsFileOutputReader(TextReader reader) { - PopulateAsync(reader); + Populate(reader); } - private void PopulateAsync(TextReader reader) + private void Populate(TextReader reader) { int overflowLength = 0; var buffer = new char[4096]; // must be large enough to hold at least one line of output