diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e0dfafc199b..f55b3212448 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,5 +23,6 @@ } }, "postCreateCommand": "dotnet restore src/Test && dotnet restore src/Runner.PluginHost", - "remoteUser": "vscode" + "remoteUser": "vscode", + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"] } diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 8806a20fa8f..788b566de5b 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -91,11 +91,18 @@ function acquireExternalTool() { elif [[ "$download_basename" == *.tar.gz ]]; then # Extract the tar gz. echo "Testing tar gz" - tar xzf "$download_target" -C "$download_dir" > /dev/null || checkRC 'tar' + # Extract to a temp dir first in case the target filesystem + # doesn't support symlinks (e.g. VirtioFS/fakeowner on macOS). + local tmp_precache_dir + tmp_precache_dir=$(mktemp -d) + tar xzf "$download_target" -C "$tmp_precache_dir" > /dev/null || { rm -rf "$tmp_precache_dir"; checkRC 'tar'; } + cp -aL "$tmp_precache_dir/." "$download_dir/" > /dev/null || { rm -rf "$tmp_precache_dir"; checkRC 'cp'; } + rm -rf "$tmp_precache_dir" fi fi else # Extract to layout. + rm -rf "$target_dir" || checkRC 'rm' mkdir -p "$target_dir" || checkRC 'mkdir' local nested_dir="" if [[ "$download_basename" == *.zip ]]; then @@ -114,7 +121,13 @@ function acquireExternalTool() { elif [[ "$download_basename" == *.tar.gz ]]; then # Extract the tar gz. echo "Extracting tar gz to layout" - tar xzf "$download_target" -C "$target_dir" > /dev/null || checkRC 'tar' + # Extract to a temp dir first in case the target filesystem + # doesn't support symlinks (e.g. VirtioFS/fakeowner on macOS). + local tmp_layout_dir + tmp_layout_dir=$(mktemp -d) + tar xzf "$download_target" -C "$tmp_layout_dir" > /dev/null || { rm -rf "$tmp_layout_dir"; checkRC 'tar'; } + cp -aL "$tmp_layout_dir/." "$target_dir/" > /dev/null || { rm -rf "$tmp_layout_dir"; checkRC 'cp'; } + rm -rf "$tmp_layout_dir" # Capture the nested directory path if the fix_nested_dir flag is set. if [[ "$fix_nested_dir" == "fix_nested_dir" ]]; then diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 583958981a9..df338f3e9ba 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -247,6 +247,21 @@ public static class Hooks public static readonly string ContainerHooksPath = "ACTIONS_RUNNER_CONTAINER_HOOKS"; } + public static class FuseMount + { + public static readonly string DriverPath = "/tmp/codespaces-fuse-driver"; + public static readonly string DriverApiUrl = "https://online.dev.core.vsengsaas.visualstudio.com/api/v1/Agents/vsoagentlinux"; + public static readonly string DriverTempDir = "/tmp/codespaces-agent"; + public static readonly string DriverTempZip = "/tmp/codespaces-agent.zip"; + public static readonly string DriverInnerPath = "storage-driver-rs-abi-7-19/codespaces-fuse-driver"; + public static readonly string BaseCacheDir = "/tmp/cache"; + public static readonly string CacheFile = System.IO.Path.Combine(BaseCacheDir, "cache"); + public static readonly string WritableFile = System.IO.Path.Combine(BaseCacheDir, "writable"); + public static readonly string LayersPath = System.IO.Path.Combine(BaseCacheDir, "layers.json"); + public static readonly int FuseDriverReadyTimeoutMs = 30000; + public static readonly int FuseDriverPollIntervalMs = 200; + } + public static class Path { public static readonly string ActionsDirectory = "_actions"; diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index c210ebeb80a..730d94e36b7 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -50,6 +50,9 @@ public sealed class JobExtension : RunnerService, IJobExtension private Task _diskSpaceCheckTask = null; private CancellationTokenSource _serviceConnectivityCheckToken = new(); private Task _serviceConnectivityCheckTask = null; + private int _fusePid = -1; + private string _fuseMountPath; + private string _fuseMountLocation; // Download all required actions. // Make sure all condition inputs are valid. @@ -197,6 +200,21 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } context.SetGitHubContext("workspace", githubWorkspace); + // Run FUSE driver to mount blob store + const bool mountWorkDirectory = true; // TODO of course only if there's actually a "mount" on the job + if (mountWorkDirectory) + { + if (Constants.Runner.Platform != Constants.OSPlatform.Linux) + { + context.Error("Mounting drive only supported on Linux runners."); + } + else + { + context.Output($"Mounting workflow directory {githubWorkspace} to blob store"); + await MountWithFuseAsync(context, githubWorkspace); + } + } + // Temporary hack for GHES alpha var configurationStore = HostContext.GetService(); var runnerSettings = configurationStore.GetSettings(); @@ -536,6 +554,24 @@ public async Task FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRe context.Start(); context.Debug("Starting: Complete job"); + // Unmount FUSE drive and stop driver if we started one. + if (_fusePid > 0) + { + context.Output($"Unmounting workflow directory {_fuseMountPath}"); + var umountInvoker = HostContext.CreateService(); + await umountInvoker.ExecuteAsync(string.Empty, "sudo", $"umount {_fuseMountPath}", null, requireExitCodeZero: false, cancellationToken: CancellationToken.None); + + context.Output($"Stopping FUSE driver (pid {_fusePid})"); + var killInvoker = HostContext.CreateService(); + await killInvoker.ExecuteAsync(string.Empty, "sudo", $"kill {_fusePid}", null, requireExitCodeZero: false, cancellationToken: CancellationToken.None); + + if (!string.IsNullOrEmpty(_fuseMountLocation) && Directory.Exists(_fuseMountLocation)) + { + var rmInvoker = HostContext.CreateService(); + await rmInvoker.ExecuteAsync(string.Empty, "rm", $"-rf {_fuseMountLocation}", null, requireExitCodeZero: false, cancellationToken: CancellationToken.None); + } + } + Trace.Info("Initialize Env context"); #if OS_WINDOWS @@ -950,6 +986,196 @@ private Dictionary SnapshotProcesses() return snapshot; } + private async Task EnsureFuseDriverAsync(IExecutionContext context) + { + if (File.Exists(Constants.FuseMount.DriverPath)) + { + context.Output($"FUSE driver already present at {Constants.FuseMount.DriverPath}."); + return; + } + + context.Output("Installing Codespaces FUSE driver..."); + + // Fetch the asset download URL from the API. + context.Output("Fetching driver download URL..."); + string downloadUrl; + using (var httpClient = new HttpClient(HostContext.CreateHttpClientHandler())) + { + var json = await httpClient.GetStringAsync(Constants.FuseMount.DriverApiUrl); + var response = StringUtil.ConvertFromJson>(json); + if (response == null || !response.TryGetValue("assetUri", out var assetUri) || string.IsNullOrEmpty(assetUri?.ToString())) + { + throw new InvalidOperationException("Failed to get download URL from API response."); + } + downloadUrl = assetUri.ToString(); + } + + context.Output($"Downloading from: {downloadUrl}"); + + // Clean up any previous attempts. + if (Directory.Exists(Constants.FuseMount.DriverTempDir)) + { + var rmInvoker = HostContext.CreateService(); + await rmInvoker.ExecuteAsync(string.Empty, "rm", $"-rf {Constants.FuseMount.DriverTempDir}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + } + if (File.Exists(Constants.FuseMount.DriverTempZip)) + { + File.Delete(Constants.FuseMount.DriverTempZip); + } + + // Download the zip archive. + var wgetInvoker = HostContext.CreateService(); + await wgetInvoker.ExecuteAsync(string.Empty, "wget", $"-O {Constants.FuseMount.DriverTempZip} {downloadUrl}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + + // Extract the zip. + context.Output("Extracting driver archive..."); + var unzipInvoker = HostContext.CreateService(); + await unzipInvoker.ExecuteAsync(string.Empty, "unzip", $"{Constants.FuseMount.DriverTempZip} -d {Constants.FuseMount.DriverTempDir}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + + // Verify the driver exists at the expected inner path. + var driverSourcePath = Path.Combine(Constants.FuseMount.DriverTempDir, Constants.FuseMount.DriverInnerPath); + if (!File.Exists(driverSourcePath)) + { + throw new FileNotFoundException($"Driver not found at expected path: {driverSourcePath}"); + } + + // Copy to system location and make executable. + context.Output($"Installing driver to {Constants.FuseMount.DriverPath}..."); + File.Copy(driverSourcePath, Constants.FuseMount.DriverPath, overwrite: true); +#pragma warning disable CA1416 // mounting is only ever reached on Linux + File.SetUnixFileMode(Constants.FuseMount.DriverPath, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute); +#pragma warning restore CA1416 + + // Clean up temp files. + var cleanupInvoker = HostContext.CreateService(); + await cleanupInvoker.ExecuteAsync(string.Empty, "rm", $"-rf {Constants.FuseMount.DriverTempDir} {Constants.FuseMount.DriverTempZip}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + + context.Output("Codespaces FUSE driver installed successfully."); + } + + private async Task MountWithFuseAsync(IExecutionContext context, string mountPath) + { + await EnsureFuseDriverAsync(context); + + // Create a unique temporary directory for the FUSE driver to use as its mount location + // before the loop device is surfaced to the OS. + var fuseMountLocation = Path.Combine(Constants.FuseMount.BaseCacheDir, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(fuseMountLocation); + + var fuseArgs = string.Join(" ", + "start", + "--layers-file", Constants.FuseMount.LayersPath, + "--file-cache-path", Constants.FuseMount.CacheFile, + "--writable-file-path", Constants.FuseMount.WritableFile, + "--fuse-mount-directory", fuseMountLocation, + "--log-base", "storage-v2-mount", + "--log-location", Constants.FuseMount.BaseCacheDir); + + // Start the FUSE driver as a detached background process. + var fuseStartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "sudo", + Arguments = $"{Constants.FuseMount.DriverPath} {fuseArgs}", + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = false, + RedirectStandardError = false, + }; + + var fuseProcess = System.Diagnostics.Process.Start(fuseStartInfo) + ?? throw new InvalidOperationException("Failed to start FUSE driver process."); + _fusePid = fuseProcess.Id; + _fuseMountPath = mountPath; + _fuseMountLocation = fuseMountLocation; + fuseProcess.Dispose(); // release .NET handle; we track only the PID + + // Poll losetup until the FUSE driver surfaces a loop device backed by fuseMountLocation. + Trace.Info($"Waiting for FUSE driver to be ready at {fuseMountLocation}"); + string deviceName = null; + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(Constants.FuseMount.FuseDriverReadyTimeoutMs)); + while (!cts.Token.IsCancellationRequested) + { + deviceName = await GetFuseDeviceNameAsync(fuseMountLocation); + if (deviceName != null) + { + break; + } + await Task.Delay(Constants.FuseMount.FuseDriverPollIntervalMs, cts.Token).ContinueWith(_ => { }); + } + + if (string.IsNullOrEmpty(deviceName)) + { + throw new TimeoutException($"Timed out waiting for FUSE driver to expose a loop device for {fuseMountLocation}."); + } + + Trace.Info($"FUSE driver ready. Loop device: {deviceName}"); + + // Ensure the target mount directory exists, then mount. + var mkdirInvoker = HostContext.CreateService(); + await mkdirInvoker.ExecuteAsync( + workingDirectory: string.Empty, + fileName: "sudo", + arguments: $"mkdir -p {mountPath}", + environment: null, + requireExitCodeZero: true, + cancellationToken: context.CancellationToken); + + var mountInvoker = HostContext.CreateService(); + await mountInvoker.ExecuteAsync( + workingDirectory: string.Empty, + fileName: "sudo", + arguments: $"mount {deviceName} {mountPath}", + environment: null, + requireExitCodeZero: true, + cancellationToken: context.CancellationToken); + + context.Output($"Workflow directory {mountPath} mounted from loop device {deviceName}"); + } + + private async Task GetFuseDeviceNameAsync(string fuseMountLocation) + { + try + { + var output = new System.Text.StringBuilder(); + var invoker = HostContext.CreateService(); + invoker.OutputDataReceived += (_, e) => + { + if (!string.IsNullOrEmpty(e.Data)) + { + output.AppendLine(e.Data); + } + }; + await invoker.ExecuteAsync( + workingDirectory: string.Empty, + fileName: "losetup", + arguments: "--output NAME,BACK-FILE", + environment: null, + requireExitCodeZero: false, + cancellationToken: CancellationToken.None); + + foreach (var line in output.ToString().Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + if (line.Contains(fuseMountLocation)) + { + // Format: "/dev/loop0 /path/to/back-file" + var parts = line.Trim().Split((char[])null, 2, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 1) + { + return parts[0]; + } + } + } + } + catch (Exception ex) + { + Trace.Warning($"GetFuseDeviceName: exception - {ex.Message}"); + } + return null; + } + private static void ValidateJobContainer(JobContainer container) { if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.RequireJobContainer)) && container == null)