[experiment] run FUSE driver and mount workspace during job setup

.devcontainer/devcontainer.json

@@ -23,5 +23,6 @@
     }
   },
   "postCreateCommand": "dotnet restore src/Test && dotnet restore src/Runner.PluginHost",
-  "remoteUser": "vscode"
+  "remoteUser": "vscode",
+  "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"]
 }

src/Misc/externals.sh

@@ -91,11 +91,18 @@ function acquireExternalTool() {
             elif [[ "$download_basename" == *.tar.gz ]]; then
                 # Extract the tar gz.
                 echo "Testing tar gz"
-                tar xzf "$download_target" -C "$download_dir" > /dev/null || checkRC 'tar'
+                # Extract to a temp dir first in case the target filesystem
+                # doesn't support symlinks (e.g. VirtioFS/fakeowner on macOS).
+                local tmp_precache_dir
+                tmp_precache_dir=$(mktemp -d)
+                tar xzf "$download_target" -C "$tmp_precache_dir" > /dev/null || { rm -rf "$tmp_precache_dir"; checkRC 'tar'; }
+                cp -aL "$tmp_precache_dir/." "$download_dir/" > /dev/null || { rm -rf "$tmp_precache_dir"; checkRC 'cp'; }
+                rm -rf "$tmp_precache_dir"
             fi
         fi
     else
         # Extract to layout.
+        rm -rf "$target_dir" || checkRC 'rm'
         mkdir -p "$target_dir" || checkRC 'mkdir'
         local nested_dir=""
         if [[ "$download_basename" == *.zip ]]; then
@@ -114,7 +121,13 @@ function acquireExternalTool() {
         elif [[ "$download_basename" == *.tar.gz ]]; then
             # Extract the tar gz.
             echo "Extracting tar gz to layout"
-            tar xzf "$download_target" -C "$target_dir" > /dev/null || checkRC 'tar'
+            # Extract to a temp dir first in case the target filesystem
+            # doesn't support symlinks (e.g. VirtioFS/fakeowner on macOS).
+            local tmp_layout_dir
+            tmp_layout_dir=$(mktemp -d)
+            tar xzf "$download_target" -C "$tmp_layout_dir" > /dev/null || { rm -rf "$tmp_layout_dir"; checkRC 'tar'; }
+            cp -aL "$tmp_layout_dir/." "$target_dir/" > /dev/null || { rm -rf "$tmp_layout_dir"; checkRC 'cp'; }
+            rm -rf "$tmp_layout_dir"
 
             # Capture the nested directory path if the fix_nested_dir flag is set.
             if [[ "$fix_nested_dir" == "fix_nested_dir" ]]; then

src/Runner.Common/Constants.cs

@@ -247,6 +247,21 @@ public static class Hooks
             public static readonly string ContainerHooksPath = "ACTIONS_RUNNER_CONTAINER_HOOKS";
         }
 
+        public static class FuseMount
+        {
+            public static readonly string DriverPath = "/tmp/codespaces-fuse-driver";
+            public static readonly string DriverApiUrl = "https://online.dev.core.vsengsaas.visualstudio.com/api/v1/Agents/vsoagentlinux";
+            public static readonly string DriverTempDir = "/tmp/codespaces-agent";
+            public static readonly string DriverTempZip = "/tmp/codespaces-agent.zip";
+            public static readonly string DriverInnerPath = "storage-driver-rs-abi-7-19/codespaces-fuse-driver";
+            public static readonly string BaseCacheDir = "/tmp/cache";
+            public static readonly string CacheFile = System.IO.Path.Combine(BaseCacheDir, "cache");
+            public static readonly string WritableFile = System.IO.Path.Combine(BaseCacheDir, "writable");
+            public static readonly string LayersPath = System.IO.Path.Combine(BaseCacheDir, "layers.json");
+            public static readonly int FuseDriverReadyTimeoutMs = 30000;
+            public static readonly int FuseDriverPollIntervalMs = 200;
+        }
+
         public static class Path
         {
             public static readonly string ActionsDirectory = "_actions";

src/Runner.Worker/JobExtension.cs

@@ -50,6 +50,9 @@ public sealed class JobExtension : RunnerService, IJobExtension
         private Task _diskSpaceCheckTask = null;
         private CancellationTokenSource _serviceConnectivityCheckToken = new();
         private Task _serviceConnectivityCheckTask = null;
+        private int _fusePid = -1;
+        private string _fuseMountPath;
+        private string _fuseMountLocation;
 
         // Download all required actions.
         // Make sure all condition inputs are valid.
@@ -197,6 +200,21 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
                     }
                     context.SetGitHubContext("workspace", githubWorkspace);
 
+                    // Run FUSE driver to mount blob store
+                    const bool mountWorkDirectory = true; // TODO of course only if there's actually a "mount" on the job
+                    if (mountWorkDirectory)
+                    {
+                        if (Constants.Runner.Platform != Constants.OSPlatform.Linux)
+                        {
+                            context.Error("Mounting drive only supported on Linux runners.");
+                        }
+                        else
+                        {
+                            context.Output($"Mounting workflow directory {githubWorkspace} to blob store");
+                            await MountWithFuseAsync(context, githubWorkspace);
+                        }
+                    }
+
                     // Temporary hack for GHES alpha
                     var configurationStore = HostContext.GetService<IConfigurationStore>();
                     var runnerSettings = configurationStore.GetSettings();
@@ -536,6 +554,24 @@ public async Task FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRe
                     context.Start();
                     context.Debug("Starting: Complete job");
 
+                    // Unmount FUSE drive and stop driver if we started one.
+                    if (_fusePid > 0)
+                    {
+                        context.Output($"Unmounting workflow directory {_fuseMountPath}");
+                        var umountInvoker = HostContext.CreateService<IProcessInvoker>();
+                        await umountInvoker.ExecuteAsync(string.Empty, "sudo", $"umount {_fuseMountPath}", null, requireExitCodeZero: false, cancellationToken: CancellationToken.None);
+
+                        context.Output($"Stopping FUSE driver (pid {_fusePid})");
+                        var killInvoker = HostContext.CreateService<IProcessInvoker>();
+                        await killInvoker.ExecuteAsync(string.Empty, "sudo", $"kill {_fusePid}", null, requireExitCodeZero: false, cancellationToken: CancellationToken.None);
+
+                        if (!string.IsNullOrEmpty(_fuseMountLocation) && Directory.Exists(_fuseMountLocation))
+                        {
+                            var rmInvoker = HostContext.CreateService<IProcessInvoker>();
+                            await rmInvoker.ExecuteAsync(string.Empty, "rm", $"-rf {_fuseMountLocation}", null, requireExitCodeZero: false, cancellationToken: CancellationToken.None);
+                        }
+                    }
+
                     Trace.Info("Initialize Env context");
 
 #if OS_WINDOWS
@@ -950,6 +986,196 @@ private Dictionary<int, Process> SnapshotProcesses()
             return snapshot;
         }
 
+        private async Task EnsureFuseDriverAsync(IExecutionContext context)
+        {
+            if (File.Exists(Constants.FuseMount.DriverPath))
+            {
+                context.Output($"FUSE driver already present at {Constants.FuseMount.DriverPath}.");
+                return;
+            }
+
+            context.Output("Installing Codespaces FUSE driver...");
+
+            // Fetch the asset download URL from the API.
+            context.Output("Fetching driver download URL...");
+            string downloadUrl;
+            using (var httpClient = new HttpClient(HostContext.CreateHttpClientHandler()))
+            {
+                var json = await httpClient.GetStringAsync(Constants.FuseMount.DriverApiUrl);
+                var response = StringUtil.ConvertFromJson<System.Collections.Generic.Dictionary<string, object>>(json);
+                if (response == null || !response.TryGetValue("assetUri", out var assetUri) || string.IsNullOrEmpty(assetUri?.ToString()))
+                {
+                    throw new InvalidOperationException("Failed to get download URL from API response.");
+                }
+                downloadUrl = assetUri.ToString();
+            }
+
+            context.Output($"Downloading from: {downloadUrl}");
+
+            // Clean up any previous attempts.
+            if (Directory.Exists(Constants.FuseMount.DriverTempDir))
+            {
+                var rmInvoker = HostContext.CreateService<IProcessInvoker>();
+                await rmInvoker.ExecuteAsync(string.Empty, "rm", $"-rf {Constants.FuseMount.DriverTempDir}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken);
+            }
+            if (File.Exists(Constants.FuseMount.DriverTempZip))
+            {
+                File.Delete(Constants.FuseMount.DriverTempZip);
+            }
+
+            // Download the zip archive.
+            var wgetInvoker = HostContext.CreateService<IProcessInvoker>();
+            await wgetInvoker.ExecuteAsync(string.Empty, "wget", $"-O {Constants.FuseMount.DriverTempZip} {downloadUrl}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken);
+
+            // Extract the zip.
+            context.Output("Extracting driver archive...");
+            var unzipInvoker = HostContext.CreateService<IProcessInvoker>();
+            await unzipInvoker.ExecuteAsync(string.Empty, "unzip", $"{Constants.FuseMount.DriverTempZip} -d {Constants.FuseMount.DriverTempDir}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken);
+
+            // Verify the driver exists at the expected inner path.
+            var driverSourcePath = Path.Combine(Constants.FuseMount.DriverTempDir, Constants.FuseMount.DriverInnerPath);
+            if (!File.Exists(driverSourcePath))
+            {
+                throw new FileNotFoundException($"Driver not found at expected path: {driverSourcePath}");
+            }
+
+            // Copy to system location and make executable.
+            context.Output($"Installing driver to {Constants.FuseMount.DriverPath}...");
+            File.Copy(driverSourcePath, Constants.FuseMount.DriverPath, overwrite: true);
+#pragma warning disable CA1416 // mounting is only ever reached on Linux
+            File.SetUnixFileMode(Constants.FuseMount.DriverPath,
+                UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
+                UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
+                UnixFileMode.OtherRead | UnixFileMode.OtherExecute);
+#pragma warning restore CA1416
+
+            // Clean up temp files.
+            var cleanupInvoker = HostContext.CreateService<IProcessInvoker>();
+            await cleanupInvoker.ExecuteAsync(string.Empty, "rm", $"-rf {Constants.FuseMount.DriverTempDir} {Constants.FuseMount.DriverTempZip}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken);
+
+            context.Output("Codespaces FUSE driver installed successfully.");
+        }
+
+        private async Task MountWithFuseAsync(IExecutionContext context, string mountPath)
+        {
+            await EnsureFuseDriverAsync(context);
+
+            // Create a unique temporary directory for the FUSE driver to use as its mount location
+            // before the loop device is surfaced to the OS.
+            var fuseMountLocation = Path.Combine(Constants.FuseMount.BaseCacheDir, Guid.NewGuid().ToString("N"));
+            Directory.CreateDirectory(fuseMountLocation);
+
+            var fuseArgs = string.Join(" ",
+                "start",
+                "--layers-file", Constants.FuseMount.LayersPath,
+                "--file-cache-path", Constants.FuseMount.CacheFile,
+                "--writable-file-path", Constants.FuseMount.WritableFile,
+                "--fuse-mount-directory", fuseMountLocation,
+                "--log-base", "storage-v2-mount",
+                "--log-location", Constants.FuseMount.BaseCacheDir);
+
+            // Start the FUSE driver as a detached background process.
+            var fuseStartInfo = new System.Diagnostics.ProcessStartInfo
+            {
+                FileName = "sudo",
+                Arguments = $"{Constants.FuseMount.DriverPath} {fuseArgs}",
+                UseShellExecute = false,
+                CreateNoWindow = true,
+                RedirectStandardOutput = false,
+                RedirectStandardError = false,
+            };
+
+            var fuseProcess = System.Diagnostics.Process.Start(fuseStartInfo)
+                ?? throw new InvalidOperationException("Failed to start FUSE driver process.");
+            _fusePid = fuseProcess.Id;
+            _fuseMountPath = mountPath;
+            _fuseMountLocation = fuseMountLocation;
+            fuseProcess.Dispose(); // release .NET handle; we track only the PID
+
+            // Poll losetup until the FUSE driver surfaces a loop device backed by fuseMountLocation.
+            Trace.Info($"Waiting for FUSE driver to be ready at {fuseMountLocation}");
+            string deviceName = null;
+            using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(Constants.FuseMount.FuseDriverReadyTimeoutMs));
+            while (!cts.Token.IsCancellationRequested)
+            {
+                deviceName = await GetFuseDeviceNameAsync(fuseMountLocation);
+                if (deviceName != null)
+                {
+                    break;
+                }
+                await Task.Delay(Constants.FuseMount.FuseDriverPollIntervalMs, cts.Token).ContinueWith(_ => { });
+            }
+
+            if (string.IsNullOrEmpty(deviceName))
+            {
+                throw new TimeoutException($"Timed out waiting for FUSE driver to expose a loop device for {fuseMountLocation}.");
+            }
+
+            Trace.Info($"FUSE driver ready. Loop device: {deviceName}");
+
+            // Ensure the target mount directory exists, then mount.
+            var mkdirInvoker = HostContext.CreateService<IProcessInvoker>();
+            await mkdirInvoker.ExecuteAsync(
+                workingDirectory: string.Empty,
+                fileName: "sudo",
+                arguments: $"mkdir -p {mountPath}",
+                environment: null,
+                requireExitCodeZero: true,
+                cancellationToken: context.CancellationToken);
+
+            var mountInvoker = HostContext.CreateService<IProcessInvoker>();
+            await mountInvoker.ExecuteAsync(
+                workingDirectory: string.Empty,
+                fileName: "sudo",
+                arguments: $"mount {deviceName} {mountPath}",
+                environment: null,
+                requireExitCodeZero: true,
+                cancellationToken: context.CancellationToken);
+
+            context.Output($"Workflow directory {mountPath} mounted from loop device {deviceName}");
+        }
+
+        private async Task<string> GetFuseDeviceNameAsync(string fuseMountLocation)
+        {
+            try
+            {
+                var output = new System.Text.StringBuilder();
+                var invoker = HostContext.CreateService<IProcessInvoker>();
+                invoker.OutputDataReceived += (_, e) =>
+                {
+                    if (!string.IsNullOrEmpty(e.Data))
+                    {
+                        output.AppendLine(e.Data);
+                    }
+                };
+                await invoker.ExecuteAsync(
+                    workingDirectory: string.Empty,
+                    fileName: "losetup",
+                    arguments: "--output NAME,BACK-FILE",
+                    environment: null,
+                    requireExitCodeZero: false,
+                    cancellationToken: CancellationToken.None);
+
+                foreach (var line in output.ToString().Split('\n', StringSplitOptions.RemoveEmptyEntries))
+                {
+                    if (line.Contains(fuseMountLocation))
+                    {
+                        // Format: "/dev/loop0   /path/to/back-file"
+                        var parts = line.Trim().Split((char[])null, 2, StringSplitOptions.RemoveEmptyEntries);
+                        if (parts.Length >= 1)
+                        {
+                            return parts[0];
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Trace.Warning($"GetFuseDeviceName: exception - {ex.Message}");
+            }
+            return null;
+        }
+
         private static void ValidateJobContainer(JobContainer container)
         {
             if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.RequireJobContainer)) && container == null)