diff --git a/README.md b/README.md index 538f4d9..86cd542 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This project provides plugin implementations for the experimental [MSBuild Proje ## Usage -This feature requires Visual Studio 17.9 or later. +This feature requires Visual Studio 17.9 or later and assumes the repo uses Git or Unity Version Control. To enable caching, simply add a `` for the desired cache implementation and set various properties to configure it. @@ -32,7 +32,7 @@ Here is an example if you're using NuGet's [Central Package Management](https:// ``` -For repos using C++, you will need to add the projects to a packages.config and import the props/targets files directly. +For repos using C++, you will need to add the projects to a packages.config and import the props/targets files directly. For caching to work, the target solution needs to be above its projects. `Directory.Build.props`: ```xml diff --git a/src/Common.Tests/SourceControl/GitFileHashProviderTest.cs b/src/Common.Tests/SourceControl/GitFileHashProviderTest.cs index 09e3955..aa99381 100644 Binary files a/src/Common.Tests/SourceControl/GitFileHashProviderTest.cs and b/src/Common.Tests/SourceControl/GitFileHashProviderTest.cs differ diff --git a/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs new file mode 100644 index 0000000..6301fbc --- /dev/null +++ b/src/Common.Tests/SourceControl/UnityVersionControlFileHashProviderTest.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using BuildXL.Utilities; +using Microsoft.MSBuildCache.SourceControl.UnityVersionControl; +using Microsoft.MSBuildCache.Tests.Mocks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.MSBuildCache.Tests.SourceControl; + +[TestClass] +public class UnityVersionControlFileHashProviderTests +{ + private static readonly byte[] FakeHash = { 0, 1, 2, 3, 4 }; + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. Justification: Always set by MSTest + public TestContext TestContext { get; set; } +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + + private static Task FakeHasher(List filesToRehash, Dictionary fileHashes) + { + foreach (string file in filesToRehash) + { + fileHashes[file] = FakeHash; + } + + return Task.CompletedTask; + } + + [TestMethod, Timeout(10000)] + public async Task ParseCmLsFiles() + { + const string repoRoot = @"C:\work\MSBuildCacheTest"; + // This has two modified and one untracked files + const string lsFilesOutput = "c:\\work\\MSBuildCacheTest\tFZMuOF2WDemh7irROkxyWw==\nc:\\work\\MSBuildCacheTest\\foo.txt\t9nwry/z6MPzLNvctyiKoFw==\nc:\\work\\MSBuildCacheTest\\bar.txt\t"; + + UnityVersionControlFileHashProvider unityFileHashProvider = new(NullPluginLogger.Instance); + Dictionary hashes = await unityFileHashProvider.ParseUnityLsFiles(new StringReader(lsFilesOutput), FakeHasher); + int filesExpected = 3; + Assert.AreEqual(filesExpected, hashes.Count, $"should be {filesExpected} files in this output"); + string barPath = Path.Combine(repoRoot, @"bar.txt").ToUpperInvariant(); + Assert.AreEqual(FakeHash, hashes[barPath], $"bytes of {barPath} should be {FakeHash} since it should have gotten hashed by the FakeHasher"); + Assert.AreEqual("0001020304", hashes[Path.Combine(repoRoot, "bar.txt")].ToHex()); + } + + [TestMethod, Timeout(10000)] + public async Task ParseRealCmLsFiles() + { + const string repoRoot = @"C:\work\devroot"; + UnityVersionControlFileHashProvider unityFileHashProvider = new(NullPluginLogger.Instance); + var dict = await unityFileHashProvider.GetFileHashesAsync(repoRoot, CancellationToken.None); + int filesExpected = 116921; + Assert.AreEqual(filesExpected, dict.Count, $"should be {filesExpected} files in this output"); + } +} \ No newline at end of file diff --git a/src/Common/MSBuildCachePluginBase.cs b/src/Common/MSBuildCachePluginBase.cs index c9688f3..43d1852 100644 --- a/src/Common/MSBuildCachePluginBase.cs +++ b/src/Common/MSBuildCachePluginBase.cs @@ -33,6 +33,7 @@ using Microsoft.MSBuildCache.Hashing; using Microsoft.MSBuildCache.Parsing; using Microsoft.MSBuildCache.SourceControl; +using Microsoft.MSBuildCache.SourceControl.UnityVersionControl; namespace Microsoft.MSBuildCache; @@ -44,6 +45,18 @@ public abstract class MSBuildCachePluginBase : MSBuildCachePluginBase : ProjectCachePluginBase, IAsyncDisposable where TPluginSettings : PluginSettings { + private class RepoInfo + { + public string Root { get; set; } + public VersionControl VersionControl { get; set; } + + public RepoInfo(string root, VersionControl versionControl) + { + Root = root; + VersionControl = versionControl; + } + } + enum VersionControl { Git, UVCS }; private static readonly string PluginAssemblyDirectory = Path.GetDirectoryName(typeof(MSBuildCachePluginBase).Assembly.Location)!; private static readonly SemaphoreSlim SinglePluginInstanceLock = new(1, 1); @@ -53,6 +66,7 @@ public abstract class MSBuildCachePluginBase : ProjectCachePlug private string? _repoRoot; private string? _buildId; + private VersionControl? _versionControl; // Set if we've received any file access report. Ideally MSBuild would tell us in the CacheContext private bool _hasHadFileAccessReport; @@ -214,12 +228,13 @@ public override Task BeginBuildAsync(CacheContext context, PluginLoggerBase logg private async Task BeginBuildInnerAsync(CacheContext context, PluginLoggerBase logger, CancellationToken cancellationToken) { _pluginLogger = logger; - - _repoRoot = GetRepoRoot(context, logger); - if (_repoRoot == null) + RepoInfo? repoInfo = GetRepoInfo(context, logger); + if (repoInfo == null) { return; } + _repoRoot = repoInfo.Root; + _versionControl = repoInfo.VersionControl; _buildId = GetBuildId(); @@ -1033,8 +1048,9 @@ private bool TryAcquireLock(PluginSettings settings, PluginLoggerBase logger) return true; } - private static string? GetRepoRoot(CacheContext context, PluginLoggerBase logger) + private static RepoInfo? GetRepoInfo(CacheContext context, PluginLoggerBase logger) { + RepoInfo? repoInfo = null; IEnumerable projectFilePaths = context.Graph != null ? context.Graph.EntryPointNodes.Select(node => node.ProjectInstance.FullPath) : context.GraphEntryPoints != null @@ -1044,12 +1060,10 @@ private bool TryAcquireLock(PluginSettings settings, PluginLoggerBase logger) HashSet repoRoots = new(StringComparer.OrdinalIgnoreCase); foreach (string projectFilePath in projectFilePaths) { - string? repoRoot = GetRepoRootInternal(Path.GetDirectoryName(projectFilePath)!); - - // Tolerate projects which aren't under any git repo. - if (repoRoot != null) + repoInfo = GetRepoInfoInternal(Path.GetDirectoryName(projectFilePath)!, logger); + if (repoInfo != null) { - repoRoots.Add(repoRoot); + repoRoots.Add(repoInfo.Root); } } @@ -1063,23 +1077,29 @@ private bool TryAcquireLock(PluginSettings settings, PluginLoggerBase logger) { string repoRoot = repoRoots.First(); logger.LogMessage($"Repo root: {repoRoot}"); - return repoRoot; + return repoInfo; } logger.LogWarning($"Graph contains projects from multiple git repositories. Disabling the cache. Repo roots: {string.Join(", ", repoRoots)}"); return null; - static string? GetRepoRootInternal(string path) + static RepoInfo? GetRepoInfoInternal(string path, PluginLoggerBase logger) { // Note: When using git worktrees, .git may be a file instead of a directory. string gitPath = Path.Combine(path, ".git"); if (Directory.Exists(gitPath) || File.Exists(gitPath)) { - return path; + return new RepoInfo(path, VersionControl.Git); + } + + string unityVersionControlPath = Path.Combine(path, ".plastic"); + if (Directory.Exists(unityVersionControlPath)) + { + return new RepoInfo(path, VersionControl.UVCS); } string? parentDir = Path.GetDirectoryName(path); - return parentDir != null ? GetRepoRootInternal(parentDir) : null; + return parentDir != null ? GetRepoInfoInternal(parentDir, logger) : null; } } @@ -1121,15 +1141,21 @@ void WarnIfCowNotSupportedBetweenRepoRootAndPath(string path, string pathDescrip private async Task> GetSourceControlFileHashesAsync(PluginLoggerBase logger, CancellationToken cancellationToken) { - if (_repoRoot == null) + if (_repoRoot == null || _versionControl == null) { throw new InvalidOperationException($"{nameof(_repoRoot)} was unexpectedly null"); } - logger.LogMessage("Source Control: Getting hashes"); + logger.LogMessage($"Source Control: Getting hashes from {_versionControl.Value}"); Stopwatch stopwatch = Stopwatch.StartNew(); - GitFileHashProvider hashProvider = new(logger); + ISourceControlFileHashProvider hashProvider = _versionControl switch + { + VersionControl.Git => new GitFileHashProvider(logger), + VersionControl.UVCS => new UnityVersionControlFileHashProvider(logger), + _ => throw new NotImplementedException() + }; + IReadOnlyDictionary fileHashes = await hashProvider.GetFileHashesAsync(_repoRoot, cancellationToken); logger.LogMessage($"Source Control: File hashes query took {stopwatch.ElapsedMilliseconds} ms"); diff --git a/src/Common/SourceControl/Git.cs b/src/Common/SourceControl/Git.cs index 7f94d43..3150755 100644 --- a/src/Common/SourceControl/Git.cs +++ b/src/Common/SourceControl/Git.cs @@ -1,10 +1,12 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; +using BuildXL.Utilities.Core.Tasks; using Microsoft.Build.Experimental.ProjectCache; #if NETFRAMEWORK using Process = Microsoft.MSBuildCache.SourceControl.GitProcess; @@ -95,4 +97,51 @@ static void KillProcess(Process process) } } } + + internal static Task HashObjectAsync(string basePath, List filesToRehash, Dictionary filehashes, PluginLoggerBase _logger, CancellationToken cancellationToken) + { + return RunAsync( + _logger, + workingDir: basePath, + "hash-object --stdin-paths", + async (stdin, stdout) => + { + foreach (string file in filesToRehash) + { + string? gitHashOfFile; + + if (File.Exists(file)) + { + await stdin.WriteLineAsync(file); + gitHashOfFile = await stdout.ReadLineAsync(); + + if (string.IsNullOrWhiteSpace(gitHashOfFile)) + { + _logger.LogMessage($"git hash-object returned an empty string for {file}. Forcing a cache miss by using a Guid"); + + // Guids are only 32 characters and git hashes are 40. Prepend 8 characters to match and to generally be recognizable. + gitHashOfFile = "bad00000" + Guid.NewGuid().ToString("N"); + } + } + else + { + gitHashOfFile = null; + } + + filehashes[file] = HexUtilities.HexToBytes(gitHashOfFile); + } + + return Unit.Void; + }, + (exitCode, result) => + { + if (exitCode != 0) + { + throw new SourceControlHashException("git hash-object failed with exit code " + exitCode); + } + + return result; + }, + cancellationToken); + } } \ No newline at end of file diff --git a/src/Common/SourceControl/GitFileHashProvider.cs b/src/Common/SourceControl/GitFileHashProvider.cs index 1f6f641..11d30fa 100644 --- a/src/Common/SourceControl/GitFileHashProvider.cs +++ b/src/Common/SourceControl/GitFileHashProvider.cs @@ -102,7 +102,7 @@ private async Task> GetModuleFileHashesAsync(string b _logger, workingDir: basePath, "ls-files -z -cmos --exclude-standard", - (_, stdout) => Task.Run(() => ParseGitLsFiles(basePath, stdout, (filesToRehash, fileHashes) => GitHashObjectAsync(basePath, filesToRehash, fileHashes, cancellationToken))), + (_, stdout) => Task.Run(() => ParseGitLsFiles(basePath, stdout, (filesToRehash, fileHashes) => Git.HashObjectAsync(basePath, filesToRehash, fileHashes, _logger, cancellationToken))), (exitCode, result) => { if (exitCode != 0) @@ -193,53 +193,6 @@ internal async Task> ParseGitLsFiles( return fileHashes; } - internal Task GitHashObjectAsync(string basePath, List filesToRehash, Dictionary filehashes, CancellationToken cancellationToken) - { - return Git.RunAsync( - _logger, - workingDir: basePath, - "hash-object --stdin-paths", - async (stdin, stdout) => - { - foreach (string file in filesToRehash) - { - string? gitHashOfFile; - - if (File.Exists(file)) - { - await stdin.WriteLineAsync(file); - gitHashOfFile = await stdout.ReadLineAsync(); - - if (string.IsNullOrWhiteSpace(gitHashOfFile)) - { - _logger.LogMessage($"git hash-object returned an empty string for {file}. Forcing a cache miss by using a Guid"); - - // Guids are only 32 characters and git hashes are 40. Prepend 8 characters to match and to generally be recognizable. - gitHashOfFile = "bad00000" + Guid.NewGuid().ToString("N"); - } - } - else - { - gitHashOfFile = null; - } - - filehashes[file] = HexUtilities.HexToBytes(gitHashOfFile); - } - - return Unit.Void; - }, - (exitCode, result) => - { - if (exitCode != 0) - { - throw new SourceControlHashException("git ls-files failed with exit code " + exitCode); - } - - return result; - }, - cancellationToken); - } - private Task> GetInitializedSubmodulesAsync(string repoRoot, CancellationToken cancellationToken) { string stdErrString = string.Empty; diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs new file mode 100644 index 0000000..bd91abf --- /dev/null +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControl.cs @@ -0,0 +1,84 @@ + +using System; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Experimental.ProjectCache; + +namespace Microsoft.MSBuildCache.SourceControl.UnityVersionControl +{ + internal static class UnityVersionControl + { + public static async Task RunAsync( + PluginLoggerBase logger, + string workingDir, string args, + Func> onRunning, + Func onExit, + CancellationToken cancellationToken) + { + using Process process = new(); + process.StartInfo.FileName = "cm"; // Unity Version Control command line "cm" is expected to be on the PATH + process.StartInfo.Arguments = args; + process.StartInfo.UseShellExecute = false; + process.StartInfo.CreateNoWindow = true; + process.StartInfo.WorkingDirectory = workingDir; + process.StartInfo.RedirectStandardInput = true; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = Encoding.UTF8; + + Stopwatch sw = Stopwatch.StartNew(); + + process.Start(); + + static void KillProcess(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(); + } + } + catch + { + // Swallow. This is best-effort + } + } + + using (cancellationToken.Register(() => KillProcess(process))) + { + using (StreamWriter stdin = process.StandardInput) + using (StreamReader stdout = process.StandardOutput) + using (StreamReader stderr = process.StandardError) + { + Task resultTask = Task.Run(async () => + { + return await onRunning(stdout); + }); + Task errorTask = Task.Run(() => stderr.ReadToEndAsync()); + +#if NETFRAMEWORK + process.WaitForExit(); + cancellationToken.ThrowIfCancellationRequested(); +#else + await process.WaitForExitAsync(cancellationToken); +#endif + + if (process.ExitCode == 0) + { + logger.LogMessage($"cm.exe {args} (@{process.StartInfo.WorkingDirectory}) took {sw.ElapsedMilliseconds} msec and returned {process.ExitCode}."); + } + else + { + logger.LogMessage($"cm.exe {args} (@{process.StartInfo.WorkingDirectory}) took {sw.ElapsedMilliseconds} msec and returned {process.ExitCode}. Stderr: {await errorTask}"); + } + + return onExit(process.ExitCode, await resultTask); + } + } + } + } +} diff --git a/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs new file mode 100644 index 0000000..9f9140d --- /dev/null +++ b/src/Common/SourceControl/UnityVersionControl/UnityVersionControlFileHashProvider.cs @@ -0,0 +1,173 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Experimental.ProjectCache; + +namespace Microsoft.MSBuildCache.SourceControl.UnityVersionControl +{ + internal class UnityVersionControlFileHashProvider : ISourceControlFileHashProvider + { + private readonly PluginLoggerBase _logger; + public UnityVersionControlFileHashProvider(PluginLoggerBase logger) + { + _logger = logger; + } + public async Task> GetFileHashesAsync(string repoRoot, CancellationToken cancellationToken) + { + Task> hashesTask = GetRepoFileHashesAsync(repoRoot, cancellationToken); + return await hashesTask; + } + + private async Task> GetRepoFileHashesAsync(string basePath, CancellationToken cancellationToken) + { + return await UnityVersionControl.RunAsync(_logger, workingDir: basePath, "ls -R --format=\"{path}\t{hash}\"", + async (stdout) => await ParseUnityLsFiles(stdout, (filesToRehash, fileHashes) => Git.HashObjectAsync(basePath, filesToRehash, fileHashes, _logger, cancellationToken)), + (exitCode, result) => + { + if (exitCode != 0) + { + throw new SourceControlHashException("cm ls failed with exit code " + exitCode); + } + + return result; + }, + cancellationToken); + } + + internal async Task> ParseUnityLsFiles( + TextReader cmOutput, + Func, Dictionary, Task> hasher) + { + // relativePathInRepositoryhash + using var reader = new UnityVersionContorlLsFileOutputReader(cmOutput); + var fileHashes = new Dictionary(StringComparer.OrdinalIgnoreCase); + var filesToRehash = new List(); + StringBuilder? lineSb; + while ((lineSb = reader.ReadLine()) != null) + { + int delimiterIndex = -1; + char delimiter = '\t'; + for (int i = 0; i < lineSb.Length; i++) + { + if (lineSb[i] == delimiter) + { + delimiterIndex = i; + break; + } + } + if (delimiterIndex == -1) + { + throw new InvalidDataException("Failed to split the string, missing a tab"); + } + + string file = lineSb.ToString(0, delimiterIndex); + int hashStartIndex = delimiterIndex + 1; + // Check that the line contains a hash, i.e. more than just the file path + if (hashStartIndex < lineSb.Length) + { + string hash = lineSb.ToString(hashStartIndex, lineSb.Length - hashStartIndex); + + try + { + fileHashes[file] = Convert.FromBase64String(hash); + } + catch (FormatException fe) + { + // Add the file and hash to the exception so the log is a bit easier to troubleshoot + throw new FormatException($"Failed to convert base64 hash to bytes for file: {file}, hash: {hash}", fe); + } + } + else + { + _logger.LogMessage($"{file} is missing a hash and will be rehashed."); + filesToRehash.Add(file); + } + } + + if (filesToRehash.Count > 0) + { + Stopwatch sw = Stopwatch.StartNew(); + // we could do this as new files come in just not clear it's worth it + await hasher(filesToRehash, fileHashes); + _logger.LogMessage($"{fileHashes.Count} files Rehashing {filesToRehash.Count} modified files took {sw.ElapsedMilliseconds} msec"); + } + + return fileHashes; + } + + private sealed class UnityVersionContorlLsFileOutputReader : IDisposable + { + readonly BlockingCollection _lines = new BlockingCollection(); + + public UnityVersionContorlLsFileOutputReader(TextReader reader) + { + Populate(reader); + } + + private void Populate(TextReader reader) + { + int overflowLength = 0; + var buffer = new char[4096]; // must be large enough to hold at least one line of output + while (true) + { + int readCnt = reader.Read(buffer, overflowLength, buffer.Length - overflowLength); + if (readCnt == 0) // end of stream + { + if (overflowLength > 0) + { + _lines.Add(new StringBuilder(overflowLength).Append(buffer, 0, overflowLength)); + } + _lines.CompleteAdding(); + return; + } + + readCnt += overflowLength; + int startIdx = 0, eolIdx; + while (startIdx < readCnt && (eolIdx = Array.IndexOf(buffer, '\n', startIdx)) != -1) + { + int lineLength = eolIdx - startIdx; + if (overflowLength > 0) + { + overflowLength = 0; + startIdx = 0; + } + _lines.Add(new StringBuilder(lineLength).Append(buffer, startIdx, lineLength)); + startIdx = eolIdx + 1; + } + if (startIdx < readCnt) + { + if (overflowLength > 0) // we already have some overflow left, but the line could not fit the buffer + { + throw new InvalidDataException($"Internal: cm ls output line length {readCnt - startIdx} exceeds {nameof(buffer)} size {buffer.Length}. Increase the latter."); + } + overflowLength = readCnt - startIdx; + Array.Copy(buffer, startIdx, buffer, 0, overflowLength); + } + } + } + + public StringBuilder? ReadLine() + { + while (!_lines.IsCompleted) + { + if (_lines.TryTake(out StringBuilder? result, -1)) + { + return result; + } + } + return null; + } + + public void Dispose() + { + _lines.Dispose(); + } + } + } + +}