From ddc99932bb009ec31467815ebdf6faf59918756c Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 12:16:24 -0500 Subject: [PATCH 01/16] starter comment --- src/Runner.Worker/JobExtension.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index c210ebeb80a..27e2df53247 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -184,6 +184,8 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context, message.Workspace); + // TODO run FUSE driver to mount work directory to blob storage + // Set the directory variables context.Debug("Update context data"); string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); From 8b893b0635e038518a25b1dc1ca3306c5a63e1aa Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 12:23:24 -0500 Subject: [PATCH 02/16] stub logic --- src/Runner.Worker/JobExtension.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 27e2df53247..3c495189bde 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -184,7 +184,19 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context, message.Workspace); - // TODO run FUSE driver to mount work directory to blob storage + const bool mountWorkDirectory = true; // TODO only if there's actually a "mount" on the job + if (mountWorkDirectory) + { + context.Output("Mounting workflow directory to blob store"); + if (Constants.Runner.Platform != Constants.OSPlatform.Linux) + { + context.Error("Mounting drive only supported on Linux runners."); + } + else + { + // TODO run FUSE driver + } + } // Set the directory variables context.Debug("Update context data"); From 25238a6361dfa78d51476beb2e54a13acf0f3ef9 Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 13:56:27 -0500 Subject: [PATCH 03/16] move this down --- src/Runner.Worker/JobExtension.cs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 3c495189bde..89e1b7c3345 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -184,20 +184,6 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context, message.Workspace); - const bool mountWorkDirectory = true; // TODO only if there's actually a "mount" on the job - if (mountWorkDirectory) - { - context.Output("Mounting workflow directory to blob store"); - if (Constants.Runner.Platform != Constants.OSPlatform.Linux) - { - context.Error("Mounting drive only supported on Linux runners."); - } - else - { - // TODO run FUSE driver - } - } - // Set the directory variables context.Debug("Update context data"); string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); @@ -211,6 +197,21 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } context.SetGitHubContext("workspace", githubWorkspace); + // Run FUSE driver to mount blob store + const bool mountWorkDirectory = true; // TODO only if there's actually a "mount" on the job + if (mountWorkDirectory) + { + if (Constants.Runner.Platform != Constants.OSPlatform.Linux) + { + context.Error("Mounting drive only supported on Linux runners."); + } + else + { + context.Output($"Mounting workflow directory {githubWorkspace} to blob store"); + // TODO run FUSE driver + } + } + // Temporary hack for GHES alpha var configurationStore = HostContext.GetService(); var runnerSettings = configurationStore.GetSettings(); From f5616ccc5a894f4229fc3ec64ea36b62e490c856 Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:30:50 -0500 Subject: [PATCH 04/16] DevContainer fixes (will split off) --- .devcontainer/devcontainer.json | 3 ++- src/Misc/externals.sh | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e0dfafc199b..f55b3212448 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,5 +23,6 @@ } }, "postCreateCommand": "dotnet restore src/Test && dotnet restore src/Runner.PluginHost", - "remoteUser": "vscode" + "remoteUser": "vscode", + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"] } diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 8806a20fa8f..788b566de5b 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -91,11 +91,18 @@ function acquireExternalTool() { elif [[ "$download_basename" == *.tar.gz ]]; then # Extract the tar gz. echo "Testing tar gz" - tar xzf "$download_target" -C "$download_dir" > /dev/null || checkRC 'tar' + # Extract to a temp dir first in case the target filesystem + # doesn't support symlinks (e.g. VirtioFS/fakeowner on macOS). + local tmp_precache_dir + tmp_precache_dir=$(mktemp -d) + tar xzf "$download_target" -C "$tmp_precache_dir" > /dev/null || { rm -rf "$tmp_precache_dir"; checkRC 'tar'; } + cp -aL "$tmp_precache_dir/." "$download_dir/" > /dev/null || { rm -rf "$tmp_precache_dir"; checkRC 'cp'; } + rm -rf "$tmp_precache_dir" fi fi else # Extract to layout. + rm -rf "$target_dir" || checkRC 'rm' mkdir -p "$target_dir" || checkRC 'mkdir' local nested_dir="" if [[ "$download_basename" == *.zip ]]; then @@ -114,7 +121,13 @@ function acquireExternalTool() { elif [[ "$download_basename" == *.tar.gz ]]; then # Extract the tar gz. echo "Extracting tar gz to layout" - tar xzf "$download_target" -C "$target_dir" > /dev/null || checkRC 'tar' + # Extract to a temp dir first in case the target filesystem + # doesn't support symlinks (e.g. VirtioFS/fakeowner on macOS). + local tmp_layout_dir + tmp_layout_dir=$(mktemp -d) + tar xzf "$download_target" -C "$tmp_layout_dir" > /dev/null || { rm -rf "$tmp_layout_dir"; checkRC 'tar'; } + cp -aL "$tmp_layout_dir/." "$target_dir/" > /dev/null || { rm -rf "$tmp_layout_dir"; checkRC 'cp'; } + rm -rf "$tmp_layout_dir" # Capture the nested directory path if the fix_nested_dir flag is set. if [[ "$fix_nested_dir" == "fix_nested_dir" ]]; then From 669190759117841bbf54ff1539e1225c9c35de6d Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:31:38 -0500 Subject: [PATCH 05/16] MountWithFuseAsync v0.1 --- src/Runner.Common/Constants.cs | 11 +++ src/Runner.Worker/JobExtension.cs | 117 +++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index fcb7c5b3503..d915a60eb5e 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -246,6 +246,17 @@ public static class Hooks public static readonly string ContainerHooksPath = "ACTIONS_RUNNER_CONTAINER_HOOKS"; } + public static class FuseMount + { + public static readonly string DriverPath = "/usr/local/bin/codespaces-fuse-driver"; + public static readonly string BaseCacheDir = "/cache"; + public static readonly string CacheFile = "/cache/cache"; + public static readonly string WritableFile = "/cache/writable"; + public static readonly string LayersPath = "/cache/layers.json"; + public static readonly int FuseDriverReadyTimeoutMs = 30000; + public static readonly int FuseDriverPollIntervalMs = 200; + } + public static class Path { public static readonly string ActionsDirectory = "_actions"; diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 89e1b7c3345..9a452c08d74 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -208,7 +208,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel else { context.Output($"Mounting workflow directory {githubWorkspace} to blob store"); - // TODO run FUSE driver + await MountWithFuseAsync(context, githubWorkspace); } } @@ -965,6 +965,121 @@ private Dictionary SnapshotProcesses() return snapshot; } + private async Task MountWithFuseAsync(IExecutionContext context, string mountPath) + { + // Create a unique temporary directory for the FUSE driver to use as its mount location + // before the loop device is surfaced to the OS. + var fuseMountLocation = Path.Combine(Constants.FuseMount.BaseCacheDir, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(fuseMountLocation); + + var fuseArgs = string.Join(" ", + "start", + "--layers-file", Constants.FuseMount.LayersPath, + "--file-cache-path", Constants.FuseMount.CacheFile, + "--writable-file-path", Constants.FuseMount.WritableFile, + "--fuse-mount-directory", fuseMountLocation, + "--log-base", "storage-v2-mount", + "--log-location", Constants.FuseMount.BaseCacheDir); + + // Start the FUSE driver as a detached background process. + var fuseStartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "sudo", + Arguments = $"{Constants.FuseMount.DriverPath} {fuseArgs}", + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = false, + RedirectStandardError = false, + }; + + var fuseProcess = System.Diagnostics.Process.Start(fuseStartInfo) + ?? throw new InvalidOperationException("Failed to start FUSE driver process."); + fuseProcess.Dispose(); // detached — we do not own its lifetime + + // Poll losetup until the FUSE driver surfaces a loop device backed by fuseMountLocation. + Trace.Info($"Waiting for FUSE driver to be ready at {fuseMountLocation}"); + string deviceName = null; + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(Constants.FuseMount.FuseDriverReadyTimeoutMs)); + while (!cts.Token.IsCancellationRequested) + { + deviceName = await GetFuseDeviceNameAsync(fuseMountLocation); + if (deviceName != null) + { + break; + } + await Task.Delay(Constants.FuseMount.FuseDriverPollIntervalMs, cts.Token).ContinueWith(_ => { }); + } + + if (string.IsNullOrEmpty(deviceName)) + { + throw new TimeoutException($"Timed out waiting for FUSE driver to expose a loop device for {fuseMountLocation}."); + } + + Trace.Info($"FUSE driver ready. Loop device: {deviceName}"); + + // Ensure the target mount directory exists, then mount. + var mkdirInvoker = HostContext.CreateService(); + await mkdirInvoker.ExecuteAsync( + workingDirectory: string.Empty, + fileName: "sudo", + arguments: $"mkdir -p {mountPath}", + environment: null, + requireExitCodeZero: true, + cancellationToken: context.CancellationToken); + + var mountInvoker = HostContext.CreateService(); + await mountInvoker.ExecuteAsync( + workingDirectory: string.Empty, + fileName: "sudo", + arguments: $"mount {deviceName} {mountPath}", + environment: null, + requireExitCodeZero: true, + cancellationToken: context.CancellationToken); + + context.Output($"Workflow directory {mountPath} mounted from loop device {deviceName}"); + } + + private async Task GetFuseDeviceNameAsync(string fuseMountLocation) + { + try + { + var output = new System.Text.StringBuilder(); + var invoker = HostContext.CreateService(); + invoker.OutputDataReceived += (_, e) => + { + if (!string.IsNullOrEmpty(e.Data)) + { + output.AppendLine(e.Data); + } + }; + await invoker.ExecuteAsync( + workingDirectory: string.Empty, + fileName: "losetup", + arguments: "--output NAME,BACK-FILE", + environment: null, + requireExitCodeZero: false, + cancellationToken: CancellationToken.None); + + foreach (var line in output.ToString().Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + if (line.Contains(fuseMountLocation)) + { + // Format: "/dev/loop0 /path/to/back-file" + var parts = line.Trim().Split((char[])null, 2, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 1) + { + return parts[0]; + } + } + } + } + catch (Exception ex) + { + Trace.Warning($"GetFuseDeviceName: exception - {ex.Message}"); + } + return null; + } + private static void ValidateJobContainer(JobContainer container) { if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.RequireJobContainer)) && container == null) From 5f5edc6b2fd664698fbc00af3d17ce7e19bb61b5 Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:36:50 -0500 Subject: [PATCH 06/16] clarify comment --- src/Runner.Worker/JobExtension.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 9a452c08d74..0dc3f39241f 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -198,7 +198,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context.SetGitHubContext("workspace", githubWorkspace); // Run FUSE driver to mount blob store - const bool mountWorkDirectory = true; // TODO only if there's actually a "mount" on the job + const bool mountWorkDirectory = true; // TODO of course only if there's actually a "mount" on the job if (mountWorkDirectory) { if (Constants.Runner.Platform != Constants.OSPlatform.Linux) From 1d386c162f49ccee2de3f5765545025856363b27 Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:41:10 -0500 Subject: [PATCH 07/16] use relative paths --- src/Runner.Common/Constants.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index d915a60eb5e..64d46bd3eef 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -250,9 +250,9 @@ public static class FuseMount { public static readonly string DriverPath = "/usr/local/bin/codespaces-fuse-driver"; public static readonly string BaseCacheDir = "/cache"; - public static readonly string CacheFile = "/cache/cache"; - public static readonly string WritableFile = "/cache/writable"; - public static readonly string LayersPath = "/cache/layers.json"; + public static readonly string CacheFile = System.IO.Path.Combine(BaseCacheDir, "cache"); + public static readonly string WritableFile = System.IO.Path.Combine(BaseCacheDir, "writable"); + public static readonly string LayersPath = System.IO.Path.Combine(BaseCacheDir, "layers.json"); public static readonly int FuseDriverReadyTimeoutMs = 30000; public static readonly int FuseDriverPollIntervalMs = 200; } From 0e0115ad98c0def4ae0785a93ef49f3ad5d840c3 Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:41:28 -0500 Subject: [PATCH 08/16] try /tmp --- src/Runner.Common/Constants.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 64d46bd3eef..f3eafe36e58 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -249,7 +249,7 @@ public static class Hooks public static class FuseMount { public static readonly string DriverPath = "/usr/local/bin/codespaces-fuse-driver"; - public static readonly string BaseCacheDir = "/cache"; + public static readonly string BaseCacheDir = "/tmp/cache"; public static readonly string CacheFile = System.IO.Path.Combine(BaseCacheDir, "cache"); public static readonly string WritableFile = System.IO.Path.Combine(BaseCacheDir, "writable"); public static readonly string LayersPath = System.IO.Path.Combine(BaseCacheDir, "layers.json"); From 7a1041dd2a7e6087fe261d64094ef8c3eb5df61f Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:47:02 -0500 Subject: [PATCH 09/16] actually use the mountPath --- src/Runner.Worker/JobExtension.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 0dc3f39241f..28aec4340d2 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -967,17 +967,12 @@ private Dictionary SnapshotProcesses() private async Task MountWithFuseAsync(IExecutionContext context, string mountPath) { - // Create a unique temporary directory for the FUSE driver to use as its mount location - // before the loop device is surfaced to the OS. - var fuseMountLocation = Path.Combine(Constants.FuseMount.BaseCacheDir, Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(fuseMountLocation); - var fuseArgs = string.Join(" ", "start", "--layers-file", Constants.FuseMount.LayersPath, "--file-cache-path", Constants.FuseMount.CacheFile, "--writable-file-path", Constants.FuseMount.WritableFile, - "--fuse-mount-directory", fuseMountLocation, + "--fuse-mount-directory", mountPath, "--log-base", "storage-v2-mount", "--log-location", Constants.FuseMount.BaseCacheDir); From 26544dcb3a280fa3dc91841b13050eba5f739593 Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:49:19 -0500 Subject: [PATCH 10/16] check if FUSE driver exists --- src/Runner.Worker/JobExtension.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 28aec4340d2..33f4b7af59c 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -965,14 +965,19 @@ private Dictionary SnapshotProcesses() return snapshot; } - private async Task MountWithFuseAsync(IExecutionContext context, string mountPath) + private async Task MountWithFuseAsync(IExecutionContext context, string fuseMountLocation) { + if (!File.Exists(Constants.FuseMount.DriverPath)) + { + throw new FileNotFoundException($"FUSE driver not found at '{Constants.FuseMount.DriverPath}'."); + } + var fuseArgs = string.Join(" ", "start", "--layers-file", Constants.FuseMount.LayersPath, "--file-cache-path", Constants.FuseMount.CacheFile, "--writable-file-path", Constants.FuseMount.WritableFile, - "--fuse-mount-directory", mountPath, + "--fuse-mount-directory", fuseMountLocation, "--log-base", "storage-v2-mount", "--log-location", Constants.FuseMount.BaseCacheDir); From f7e22cd2a10783d8e3c46205d7d5e5f4c019b1b4 Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:51:24 -0500 Subject: [PATCH 11/16] Revert "check if FUSE driver exists" This reverts commit 26544dcb3a280fa3dc91841b13050eba5f739593. --- src/Runner.Worker/JobExtension.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 33f4b7af59c..28aec4340d2 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -965,19 +965,14 @@ private Dictionary SnapshotProcesses() return snapshot; } - private async Task MountWithFuseAsync(IExecutionContext context, string fuseMountLocation) + private async Task MountWithFuseAsync(IExecutionContext context, string mountPath) { - if (!File.Exists(Constants.FuseMount.DriverPath)) - { - throw new FileNotFoundException($"FUSE driver not found at '{Constants.FuseMount.DriverPath}'."); - } - var fuseArgs = string.Join(" ", "start", "--layers-file", Constants.FuseMount.LayersPath, "--file-cache-path", Constants.FuseMount.CacheFile, "--writable-file-path", Constants.FuseMount.WritableFile, - "--fuse-mount-directory", fuseMountLocation, + "--fuse-mount-directory", mountPath, "--log-base", "storage-v2-mount", "--log-location", Constants.FuseMount.BaseCacheDir); From d222c4f52727a6cb31182f57e1b5aff516b779ac Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:51:27 -0500 Subject: [PATCH 12/16] Revert "actually use the mountPath" This reverts commit 7a1041dd2a7e6087fe261d64094ef8c3eb5df61f. --- src/Runner.Worker/JobExtension.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 28aec4340d2..0dc3f39241f 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -967,12 +967,17 @@ private Dictionary SnapshotProcesses() private async Task MountWithFuseAsync(IExecutionContext context, string mountPath) { + // Create a unique temporary directory for the FUSE driver to use as its mount location + // before the loop device is surfaced to the OS. + var fuseMountLocation = Path.Combine(Constants.FuseMount.BaseCacheDir, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(fuseMountLocation); + var fuseArgs = string.Join(" ", "start", "--layers-file", Constants.FuseMount.LayersPath, "--file-cache-path", Constants.FuseMount.CacheFile, "--writable-file-path", Constants.FuseMount.WritableFile, - "--fuse-mount-directory", mountPath, + "--fuse-mount-directory", fuseMountLocation, "--log-base", "storage-v2-mount", "--log-location", Constants.FuseMount.BaseCacheDir); From 30338675b9e8b3d57deca661a84c5ed33a9d831f Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:53:13 -0500 Subject: [PATCH 13/16] check if FUSE driver exists --- src/Runner.Worker/JobExtension.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 0dc3f39241f..b049acc8fc3 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -967,6 +967,11 @@ private Dictionary SnapshotProcesses() private async Task MountWithFuseAsync(IExecutionContext context, string mountPath) { + if (!File.Exists(Constants.FuseMount.DriverPath)) + { + throw new FileNotFoundException($"FUSE driver not found at '{Constants.FuseMount.DriverPath}'."); + } + // Create a unique temporary directory for the FUSE driver to use as its mount location // before the loop device is surfaced to the OS. var fuseMountLocation = Path.Combine(Constants.FuseMount.BaseCacheDir, Guid.NewGuid().ToString("N")); From f17435090235c6caffa32b1d12926d145aeaf16f Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 16:59:11 -0500 Subject: [PATCH 14/16] download FUSE driver --- src/Runner.Common/Constants.cs | 4 ++ src/Runner.Worker/JobExtension.cs | 72 +++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index f3eafe36e58..c02202e9e40 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -249,6 +249,10 @@ public static class Hooks public static class FuseMount { public static readonly string DriverPath = "/usr/local/bin/codespaces-fuse-driver"; + public static readonly string DriverApiUrl = "https://online.dev.core.vsengsaas.visualstudio.com/api/v1/Agents/vsoagentlinux"; + public static readonly string DriverTempDir = "/tmp/codespaces-agent"; + public static readonly string DriverTempZip = "/tmp/codespaces-agent.zip"; + public static readonly string DriverInnerPath = "storage-driver-rs-abi-7-19/codespaces-fuse-driver"; public static readonly string BaseCacheDir = "/tmp/cache"; public static readonly string CacheFile = System.IO.Path.Combine(BaseCacheDir, "cache"); public static readonly string WritableFile = System.IO.Path.Combine(BaseCacheDir, "writable"); diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index b049acc8fc3..0bd6216e697 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -965,13 +965,79 @@ private Dictionary SnapshotProcesses() return snapshot; } - private async Task MountWithFuseAsync(IExecutionContext context, string mountPath) + private async Task EnsureFuseDriverAsync(IExecutionContext context) { - if (!File.Exists(Constants.FuseMount.DriverPath)) + if (File.Exists(Constants.FuseMount.DriverPath)) + { + Trace.Info($"FUSE driver already present at {Constants.FuseMount.DriverPath}."); + return; + } + + context.Output("Installing Codespaces FUSE driver..."); + + // Fetch the asset download URL from the API. + context.Output("Fetching driver download URL..."); + string downloadUrl; + using (var httpClient = new HttpClient(HostContext.CreateHttpClientHandler())) + { + var json = await httpClient.GetStringAsync(Constants.FuseMount.DriverApiUrl); + var response = StringUtil.ConvertFromJson>(json); + if (response == null || !response.TryGetValue("assetUri", out var assetUri) || string.IsNullOrEmpty(assetUri?.ToString())) + { + throw new InvalidOperationException("Failed to get download URL from API response."); + } + downloadUrl = assetUri.ToString(); + } + + context.Output($"Downloading from: {downloadUrl}"); + + // Clean up any previous attempts. + if (Directory.Exists(Constants.FuseMount.DriverTempDir)) + { + var rmInvoker = HostContext.CreateService(); + await rmInvoker.ExecuteAsync(string.Empty, "rm", $"-rf {Constants.FuseMount.DriverTempDir}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + } + if (File.Exists(Constants.FuseMount.DriverTempZip)) { - throw new FileNotFoundException($"FUSE driver not found at '{Constants.FuseMount.DriverPath}'."); + File.Delete(Constants.FuseMount.DriverTempZip); } + // Download the zip archive. + var wgetInvoker = HostContext.CreateService(); + await wgetInvoker.ExecuteAsync(string.Empty, "wget", $"-O {Constants.FuseMount.DriverTempZip} {downloadUrl}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + + // Extract the zip. + context.Output("Extracting driver archive..."); + var unzipInvoker = HostContext.CreateService(); + await unzipInvoker.ExecuteAsync(string.Empty, "unzip", $"{Constants.FuseMount.DriverTempZip} -d {Constants.FuseMount.DriverTempDir}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + + // Verify the driver exists at the expected inner path. + var driverSourcePath = Path.Combine(Constants.FuseMount.DriverTempDir, Constants.FuseMount.DriverInnerPath); + if (!File.Exists(driverSourcePath)) + { + throw new FileNotFoundException($"Driver not found at expected path: {driverSourcePath}"); + } + + // Copy to system location and make executable. + context.Output($"Installing driver to {Constants.FuseMount.DriverPath}..."); + Directory.CreateDirectory(Path.GetDirectoryName(Constants.FuseMount.DriverPath)); + var cpInvoker = HostContext.CreateService(); + await cpInvoker.ExecuteAsync(string.Empty, "cp", $"{driverSourcePath} {Constants.FuseMount.DriverPath}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + + var chmodInvoker = HostContext.CreateService(); + await chmodInvoker.ExecuteAsync(string.Empty, "chmod", $"+x {Constants.FuseMount.DriverPath}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + + // Clean up temp files. + var cleanupInvoker = HostContext.CreateService(); + await cleanupInvoker.ExecuteAsync(string.Empty, "rm", $"-rf {Constants.FuseMount.DriverTempDir} {Constants.FuseMount.DriverTempZip}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + + context.Output("Codespaces FUSE driver installed successfully."); + } + + private async Task MountWithFuseAsync(IExecutionContext context, string mountPath) + { + await EnsureFuseDriverAsync(context); + // Create a unique temporary directory for the FUSE driver to use as its mount location // before the loop device is surfaced to the OS. var fuseMountLocation = Path.Combine(Constants.FuseMount.BaseCacheDir, Guid.NewGuid().ToString("N")); From 32edf7316cc06b0bb1ceafec1730b5e9f0e37a12 Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Wed, 4 Mar 2026 17:05:29 -0500 Subject: [PATCH 15/16] copy fuse driver to /tmp --- src/Runner.Common/Constants.cs | 2 +- src/Runner.Worker/JobExtension.cs | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index c02202e9e40..bd173902862 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -248,7 +248,7 @@ public static class Hooks public static class FuseMount { - public static readonly string DriverPath = "/usr/local/bin/codespaces-fuse-driver"; + public static readonly string DriverPath = "/tmp/codespaces-fuse-driver"; public static readonly string DriverApiUrl = "https://online.dev.core.vsengsaas.visualstudio.com/api/v1/Agents/vsoagentlinux"; public static readonly string DriverTempDir = "/tmp/codespaces-agent"; public static readonly string DriverTempZip = "/tmp/codespaces-agent.zip"; diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 0bd6216e697..714f26560ec 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -1020,12 +1020,13 @@ private async Task EnsureFuseDriverAsync(IExecutionContext context) // Copy to system location and make executable. context.Output($"Installing driver to {Constants.FuseMount.DriverPath}..."); - Directory.CreateDirectory(Path.GetDirectoryName(Constants.FuseMount.DriverPath)); - var cpInvoker = HostContext.CreateService(); - await cpInvoker.ExecuteAsync(string.Empty, "cp", $"{driverSourcePath} {Constants.FuseMount.DriverPath}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); - - var chmodInvoker = HostContext.CreateService(); - await chmodInvoker.ExecuteAsync(string.Empty, "chmod", $"+x {Constants.FuseMount.DriverPath}", null, requireExitCodeZero: true, cancellationToken: context.CancellationToken); + File.Copy(driverSourcePath, Constants.FuseMount.DriverPath, overwrite: true); +#pragma warning disable CA1416 // mounting is only ever reached on Linux + File.SetUnixFileMode(Constants.FuseMount.DriverPath, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute); +#pragma warning restore CA1416 // Clean up temp files. var cleanupInvoker = HostContext.CreateService(); From c7817e527868ee82979ef2d64f84a1d32f91572b Mon Sep 17 00:00:00 2001 From: Max Horstmann Date: Thu, 5 Mar 2026 08:17:58 -0500 Subject: [PATCH 16/16] stop FUSE driver at the end --- src/Runner.Worker/JobExtension.cs | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 714f26560ec..730d94e36b7 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -50,6 +50,9 @@ public sealed class JobExtension : RunnerService, IJobExtension private Task _diskSpaceCheckTask = null; private CancellationTokenSource _serviceConnectivityCheckToken = new(); private Task _serviceConnectivityCheckTask = null; + private int _fusePid = -1; + private string _fuseMountPath; + private string _fuseMountLocation; // Download all required actions. // Make sure all condition inputs are valid. @@ -551,6 +554,24 @@ public async Task FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRe context.Start(); context.Debug("Starting: Complete job"); + // Unmount FUSE drive and stop driver if we started one. + if (_fusePid > 0) + { + context.Output($"Unmounting workflow directory {_fuseMountPath}"); + var umountInvoker = HostContext.CreateService(); + await umountInvoker.ExecuteAsync(string.Empty, "sudo", $"umount {_fuseMountPath}", null, requireExitCodeZero: false, cancellationToken: CancellationToken.None); + + context.Output($"Stopping FUSE driver (pid {_fusePid})"); + var killInvoker = HostContext.CreateService(); + await killInvoker.ExecuteAsync(string.Empty, "sudo", $"kill {_fusePid}", null, requireExitCodeZero: false, cancellationToken: CancellationToken.None); + + if (!string.IsNullOrEmpty(_fuseMountLocation) && Directory.Exists(_fuseMountLocation)) + { + var rmInvoker = HostContext.CreateService(); + await rmInvoker.ExecuteAsync(string.Empty, "rm", $"-rf {_fuseMountLocation}", null, requireExitCodeZero: false, cancellationToken: CancellationToken.None); + } + } + Trace.Info("Initialize Env context"); #if OS_WINDOWS @@ -969,7 +990,7 @@ private async Task EnsureFuseDriverAsync(IExecutionContext context) { if (File.Exists(Constants.FuseMount.DriverPath)) { - Trace.Info($"FUSE driver already present at {Constants.FuseMount.DriverPath}."); + context.Output($"FUSE driver already present at {Constants.FuseMount.DriverPath}."); return; } @@ -1066,7 +1087,10 @@ private async Task MountWithFuseAsync(IExecutionContext context, string mountPat var fuseProcess = System.Diagnostics.Process.Start(fuseStartInfo) ?? throw new InvalidOperationException("Failed to start FUSE driver process."); - fuseProcess.Dispose(); // detached — we do not own its lifetime + _fusePid = fuseProcess.Id; + _fuseMountPath = mountPath; + _fuseMountLocation = fuseMountLocation; + fuseProcess.Dispose(); // release .NET handle; we track only the PID // Poll losetup until the FUSE driver surfaces a loop device backed by fuseMountLocation. Trace.Info($"Waiting for FUSE driver to be ready at {fuseMountLocation}");