Skip to content

Enable live test for remote MCP #1630

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
chidozieononiwu wants to merge 7 commits into
base: main
Choose a base branch
from
+822 −138
Draft

Enable live test for remote MCP #1630

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3ac826d
Extend testing infrastructure for Remote MCP
chidozieononiwu Jan 30, 2026
e925473
Update test resource Depolyment to allow http mode
chidozieononiwu Feb 2, 2026
00f2002
Update test-resources-post file to take UseHttpTransport
chidozieononiwu Feb 3, 2026
8fb0b51
Update live test pipeline to run test usig http transport
chidozieononiwu Feb 3, 2026
ed1ec76
Update test resource Depolyment to allow http mode
chidozieononiwu Feb 4, 2026
491094f
Add LiveServerFixture
chidozieononiwu Feb 5, 2026
82aeeb8
Update test files to use LiveServer collection
chidozieononiwu Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/ClientToolTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@

namespace Azure.Mcp.Core.LiveTests;

public class ClientToolTests(ITestOutputHelper output, TestProxyFixture testProxyFixture) : RecordedCommandTestsBase(output, testProxyFixture)
[Collection("LiveServer")]
public class ClientToolTests(ITestOutputHelper output, TestProxyFixture testProxyFixture, LiveServerFixture liveServerFixture) : RecordedCommandTestsBase(output, testProxyFixture, liveServerFixture)
{

[Fact]
Expand Down
3 changes: 2 additions & 1 deletion core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/CommandTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@

namespace Azure.Mcp.Core.LiveTests;

public class CommandTests(ITestOutputHelper output, TestProxyFixture testProxyFixture) : RecordedCommandTestsBase(output, testProxyFixture)
[Collection("LiveServer")]
public class CommandTests(ITestOutputHelper output, TestProxyFixture testProxyFixture, LiveServerFixture liveServerFixture) : RecordedCommandTestsBase(output, testProxyFixture, liveServerFixture)
{
[Fact]
public async Task Should_list_groups_by_subscription()
Expand Down
10 changes: 10 additions & 0 deletions core/Azure.Mcp.Core/tests/Azure.Mcp.Core.LiveTests/LiveServerCollection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
using Azure.Mcp.Tests.Client;
using Xunit;

namespace Azure.Mcp.Core.LiveTests
{
[CollectionDefinition("LiveServer")]
public sealed class LiveServerCollection : ICollectionFixture<LiveServerFixture>
{
}
}
3 changes: 2 additions & 1 deletion ....Mcp.Core/tests/Azure.Mcp.Core.LiveTests/RecordingFramework/RecordedCommandTestHarness.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ namespace Azure.Mcp.Core.LiveTests.RecordingFramework;
/// </summary>
/// <param name="output"></param>
/// <param name="fixture"></param>
internal sealed class RecordedCommandTestHarness(ITestOutputHelper output, TestProxyFixture fixture) : RecordedCommandTestsBase(output, fixture)
[Collection("LiveServer")]
internal sealed class RecordedCommandTestHarness(ITestOutputHelper output, TestProxyFixture fixture, LiveServerFixture liveServerFixture) : RecordedCommandTestsBase(output, fixture, liveServerFixture)
{
public TestMode DesiredMode { get; set; } = TestMode.Record;

Expand Down
13 changes: 8 additions & 5 deletions ...p.Core/tests/Azure.Mcp.Core.LiveTests/RecordingFramework/RecordedCommandTestsBaseTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Reflection;
using System.Text;
using System.Text.Json;
using Azure.Mcp.Tests.Client;
using Azure.Mcp.Tests.Client.Attributes;
using Azure.Mcp.Tests.Client.Helpers;
using Azure.Mcp.Tests.Generated.Models;
Expand Down Expand Up @@ -131,13 +132,15 @@ public sealed class RecordedCommandTestsBaseTest : IAsyncLifetime
private string TestDisplayName = string.Empty;
private readonly TemporaryAssetsPathResolver Resolver = new();
private readonly TestProxyFixture Fixture;
private readonly LiveServerFixture LiveServerFixture;
private ITestOutputHelper CollectedOutput = Substitute.For<ITestOutputHelper>();
private RecordedCommandTestHarness? DefaultHarness;

public RecordedCommandTestsBaseTest()
{
Fixture = new TestProxyFixture();
Fixture.ConfigurePathResolver(Resolver);
LiveServerFixture = new LiveServerFixture();
}

[Fact]
Expand Down Expand Up @@ -169,7 +172,7 @@ public async Task PerTestMatcherAttributeAppliesWhenPresent()
Assert.True(activeMatcher!.CompareBodies);
Assert.True(activeMatcher.IgnoreQueryOrdering);

DefaultHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
DefaultHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture, LiveServerFixture)
{
DesiredMode = TestMode.Record,
EnableDefaultSanitizerAdditions = false,
Expand All @@ -180,7 +183,7 @@ public async Task PerTestMatcherAttributeAppliesWhenPresent()
DefaultHarness.RegisterVariable("attrKey", "attrValue");
await DefaultHarness.DisposeAsync();

var playbackHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
var playbackHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture, LiveServerFixture)
{
DesiredMode = TestMode.Playback,
EnableDefaultSanitizerAdditions = false,
Expand Down Expand Up @@ -226,7 +229,7 @@ public void CustomMatcherAttributeClearsAfterExecution()
[Fact]
public async Task GlobalMatcherAndSanitizerAppliesWhenPresent()
{
DefaultHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
DefaultHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture, LiveServerFixture)
{
DesiredMode = TestMode.Record,
TestMatcher = new CustomDefaultMatcher
Expand Down Expand Up @@ -256,7 +259,7 @@ public async Task VariableSurvivesRecordPlaybackRoundtrip()
DefaultHarness.RegisterVariable("roundtrip", "value");
await DefaultHarness.DisposeAsync();

var playbackHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
var playbackHarness = new RecordedCommandTestHarness(CollectedOutput, Fixture, LiveServerFixture)
{
DesiredMode = TestMode.Playback,
};
Expand All @@ -270,7 +273,7 @@ public ValueTask InitializeAsync()
{
TestDisplayName = TestContext.Current?.Test?.TestCase?.TestCaseDisplayName ?? throw new InvalidDataException("Test case display name is not available.");

var harness = new RecordedCommandTestHarness(CollectedOutput, Fixture)
var harness = new RecordedCommandTestHarness(CollectedOutput, Fixture, LiveServerFixture)
{
DesiredMode = TestMode.Record
};
Expand Down
74 changes: 35 additions & 39 deletions core/Azure.Mcp.Core/tests/Azure.Mcp.Tests/Client/CommandTestsBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
Expand All @@ -12,14 +13,16 @@

namespace Azure.Mcp.Tests.Client;

public abstract class CommandTestsBase(ITestOutputHelper output) : IAsyncLifetime, IDisposable
[Collection("LiveServer")]
public abstract class CommandTestsBase(ITestOutputHelper output, LiveServerFixture liveServerFixture) : IAsyncLifetime, IDisposable
{
protected const string TenantNameReason = "Service principals cannot use TenantName for lookup";

protected McpClient Client { get; private set; } = default!;
protected LiveTestSettings Settings { get; set; } = default!;
protected StringBuilder FailureOutput { get; } = new();
protected ITestOutputHelper Output { get; } = output;
protected LiveServerFixture LiveServerFixture { get; } = liveServerFixture;

public string[]? CustomArguments;
public TestMode TestMode = TestMode.Live;
Expand Down Expand Up @@ -78,20 +81,8 @@ protected virtual async ValueTask LoadSettingsAsync()
}
}

protected virtual async ValueTask InitializeAsyncInternal(TestProxyFixture? proxy = null)
private Dictionary<string, string?> GetEnvironmentVariables(TestProxyFixture? proxy)
{
await LoadSettingsAsync();

string executablePath = McpTestUtilities.GetAzMcpExecutablePath();

// Use custom arguments if provided, otherwise use standard mode (debug can be enabled via environment variable)
var debugEnvVar = Environment.GetEnvironmentVariable("AZURE_MCP_TEST_DEBUG");
var enableDebug = string.Equals(debugEnvVar, "true", StringComparison.OrdinalIgnoreCase) || Settings.DebugOutput;
string[] defaultArgs = enableDebug
? ["server", "start", "--mode", "all", "--debug"]
: ["server", "start", "--mode", "all"];
var arguments = CustomArguments ?? defaultArgs;

Dictionary<string, string?> envVarDictionary = [
// Propagate playback signaling & sanitized identifiers to server process.

Expand All @@ -111,27 +102,38 @@ protected virtual async ValueTask InitializeAsyncInternal(TestProxyFixture? prox
}
}

StdioClientTransportOptions transportOptions = new()
{
Name = "Test Server",
Command = executablePath,
Arguments = arguments,
// Direct stderr to test output helper as required by task
StandardErrorLines = line => Output.WriteLine($"[MCP Server] {line}"),
EnvironmentVariables = envVarDictionary
};

if (!string.IsNullOrEmpty(Settings.TestPackage))
// Add any custom environment variables from settings
if (Settings?.EnvironmentVariables != null)
{
Environment.CurrentDirectory = Settings.SettingsDirectory;
transportOptions.Command = "npx";
transportOptions.Arguments = ["-y", Settings.TestPackage, .. arguments];
foreach (var kvp in Settings.EnvironmentVariables)
{
envVarDictionary[kvp.Key] = kvp.Value;
}
}

var clientTransport = new StdioClientTransport(transportOptions);
Output.WriteLine("Attempting to start MCP Client");
Client = await McpClient.CreateAsync(clientTransport);
Output.WriteLine("MCP client initialized successfully");
return envVarDictionary;
}

protected virtual async ValueTask InitializeAsyncInternal(TestProxyFixture? proxy = null)
{
await LoadSettingsAsync();
string executablePath = McpTestUtilities.GetAzMcpExecutablePath();

// Use custom arguments if provided, otherwise use standard mode (debug can be enabled via environment variable)
var debugEnvVar = Environment.GetEnvironmentVariable("AZURE_MCP_TEST_DEBUG");
var enableDebug = string.Equals(debugEnvVar, "true", StringComparison.OrdinalIgnoreCase) || Settings.DebugOutput;
List<string> defaultArgs = enableDebug
? ["server", "start", "--mode", "all", "--debug"]
: ["server", "start", "--mode", "all"];
var arguments = CustomArguments?.ToList() ?? defaultArgs;

LiveServerFixture.environmentVariables = GetEnvironmentVariables(proxy);
LiveServerFixture.arguments = arguments;
LiveServerFixture.output = Output;
LiveServerFixture.settings = Settings;

await LiveServerFixture.EnsureStartedAsync();
Client = LiveServerFixture.GetMcpClient();
}

protected Task<JsonElement?> CallToolAsync(string command, Dictionary<string, object?> parameters)
Expand Down Expand Up @@ -217,12 +219,6 @@ protected virtual void Dispose(bool disposing)
// No unmanaged resources to release, but if we had, we'd release them here.
// _disposableResource?.Dispose();
// _disposableResource = null;

// Handle things normally disposed in DisposeAsyncCore
if (Client is IDisposable disposable)
{
disposable.Dispose();
}
}

// Failure output may contain request and response details that should be output for failed tests.
Expand All @@ -236,6 +232,6 @@ protected virtual void Dispose(bool disposing)
// overrides should still call base.DisposeAsyncCore()
protected virtual async ValueTask DisposeAsyncCore()
{
await Client.DisposeAsync().ConfigureAwait(false);

}
}
10 changes: 10 additions & 0 deletions core/Azure.Mcp.Core/tests/Azure.Mcp.Tests/Client/LiveServerCollection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
using Azure.Mcp.Tests.Client;
using Xunit;

namespace Azure.Mcp.Tests.Client
{
[CollectionDefinition("LiveServer")]
public sealed class LiveServerCollection : ICollectionFixture<LiveServerFixture>
{
}
}
Loading
Loading