From 0267282d6b00568cf911b212ff1b4588129dc73e Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Tue, 5 May 2026 21:35:58 -0700 Subject: [PATCH 1/3] adding request.options.tools event to 3p --- .../src/extension/prompt/node/chatMLFetcher.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts index 06b1621dad469..bab0c1f179d43 100644 --- a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts +++ b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts @@ -1145,6 +1145,14 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { } this._telemetryService.sendGHTelemetryEvent('request.sent', telemetryData.properties, telemetryData.measurements); + if (request.tools) { + this._telemetryService.sendEnhancedGHTelemetryEvent('request.options.tools', { + headerRequestId: ourRequestId, + conversationId, + messagesJson: JSON.stringify(request.tools), + }, telemetryData.measurements); + } + const requestStart = Date.now(); const handle = connection.sendRequest(request, { userInitiated: !!userInitiatedRequest, turnId, requestId: ourRequestId, model: chatEndpointInfo.model, countTokens, tokenCountMax: chatEndpointInfo.maxOutputTokens, modelMaxPromptTokens: chatEndpointInfo.modelMaxPromptTokens, summarizedAtRoundId, modeChanged }, cancellationToken); @@ -1417,6 +1425,14 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { this._telemetryService.sendGHTelemetryEvent('request.sent', telemetryData.properties, telemetryData.measurements); + if (request.tools) { + this._telemetryService.sendEnhancedGHTelemetryEvent('request.options.tools', { + headerRequestId: ourRequestId, + conversationId: telemetryProperties?.conversationId, + messagesJson: JSON.stringify(request.tools), + }, telemetryData.measurements); + } + const requestStart = Date.now(); const intent = locationToIntent(location); From 4fa056e80f783eea76588b90b31c13f3be6c7877 Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Wed, 6 May 2026 10:00:32 -0700 Subject: [PATCH 2/3] multiplex in case it's too long --- .../copilot/src/extension/prompt/node/chatMLFetcher.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts index bab0c1f179d43..785b4d10e3a0e 100644 --- a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts +++ b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts @@ -32,7 +32,7 @@ import { IOTelService, ISpanHandle, SpanKind, SpanStatusCode } from '../../../pl import { IRequestLogger } from '../../../platform/requestLogger/common/requestLogger'; import { getCurrentCapturingToken } from '../../../platform/requestLogger/node/requestLogger'; import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService'; -import { ITelemetryService, TelemetryProperties } from '../../../platform/telemetry/common/telemetry'; +import { ITelemetryService, TelemetryProperties, multiplexProperties } from '../../../platform/telemetry/common/telemetry'; import { TelemetryData } from '../../../platform/telemetry/common/telemetryData'; import { isEncryptedThinkingDelta } from '../../../platform/thinking/common/thinking'; import { calculateLineRepetitionStats, isRepetitive } from '../../../util/common/anomalyDetection'; @@ -1146,11 +1146,11 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { this._telemetryService.sendGHTelemetryEvent('request.sent', telemetryData.properties, telemetryData.measurements); if (request.tools) { - this._telemetryService.sendEnhancedGHTelemetryEvent('request.options.tools', { + this._telemetryService.sendEnhancedGHTelemetryEvent('request.options.tools', multiplexProperties({ headerRequestId: ourRequestId, conversationId, messagesJson: JSON.stringify(request.tools), - }, telemetryData.measurements); + }), telemetryData.measurements); } const requestStart = Date.now(); @@ -1426,11 +1426,11 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { this._telemetryService.sendGHTelemetryEvent('request.sent', telemetryData.properties, telemetryData.measurements); if (request.tools) { - this._telemetryService.sendEnhancedGHTelemetryEvent('request.options.tools', { + this._telemetryService.sendEnhancedGHTelemetryEvent('request.options.tools', multiplexProperties({ headerRequestId: ourRequestId, conversationId: telemetryProperties?.conversationId, messagesJson: JSON.stringify(request.tools), - }, telemetryData.measurements); + }), telemetryData.measurements); } const requestStart = Date.now(); From 99e7a43de63a744c7acab00084a163555b75ae45 Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Wed, 6 May 2026 10:06:28 -0700 Subject: [PATCH 3/3] add tests --- .../chatMLFetcherResponseApiTelemetry.spec.ts | 343 ++++++++++++++++++ 1 file changed, 343 insertions(+) diff --git a/extensions/copilot/src/extension/prompt/node/test/chatMLFetcherResponseApiTelemetry.spec.ts b/extensions/copilot/src/extension/prompt/node/test/chatMLFetcherResponseApiTelemetry.spec.ts index dc810554a77bd..4feca1c578cee 100644 --- a/extensions/copilot/src/extension/prompt/node/test/chatMLFetcherResponseApiTelemetry.spec.ts +++ b/extensions/copilot/src/extension/prompt/node/test/chatMLFetcherResponseApiTelemetry.spec.ts @@ -159,8 +159,351 @@ describe('ChatMLFetcherImpl Response API telemetry', () => { }); }); +describe('ChatMLFetcherImpl request.options.tools telemetry', () => { + let disposables: DisposableStore; + let fetcher: ChatMLFetcherImpl; + let mockFetcherService: MockFetcherService; + let spyingTelemetryService: SpyingTelemetryService; + let cancellationTokenSource: CancellationTokenSource; + + beforeEach(() => { + disposables = new DisposableStore(); + cancellationTokenSource = disposables.add(new CancellationTokenSource()); + + mockFetcherService = new MockFetcherService(); + spyingTelemetryService = new SpyingTelemetryService(); + const configurationService = new InMemoryConfigurationService(new DefaultsOnlyConfigurationService()); + + const logService = new TestLogService(); + const experimentationService = new NullExperimentationService(); + + fetcher = new ChatMLFetcherImpl( + mockFetcherService as unknown as IFetcherService, + spyingTelemetryService, + new NullRequestLogger(), + logService, + new TestAuthenticationService() as unknown as IAuthenticationService, + createMockInteractionService(), + createMockChatQuotaService(), + new TestCAPIClientService() as unknown as ICAPIClientService, + createMockConversationOptions(), + configurationService, + experimentationService, + createMockPowerService(), + new InstantiationServiceBuilder([ + [IFetcherService, mockFetcherService as unknown as IFetcherService], + [ITelemetryService, spyingTelemetryService], + [ICAPIClientService, new TestCAPIClientService() as unknown as ICAPIClientService], + ]).seal() as unknown as IInstantiationService, + new NullChatWebSocketManager(), + new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '0.0.0', sessionId: 'test' })), + ); + }); + + afterEach(() => { + disposables.dispose(); + }); + + it('emits request.options.tools event when tools are present', async () => { + const endpointWithTools = createEndpointWithTools(); + mockFetcherService.queueResponse(createSuccessResponse('Hello!')); + + const opts: IFetchMLOptions = { + debugName: 'test-tools-telemetry', + messages: [{ role: Raw.ChatRole.User, content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'Use a tool' }] }], + endpoint: endpointWithTools, + location: ChatLocation.Panel, + requestOptions: {}, + finishedCb: undefined, + }; + + await fetcher.fetchMany(opts, cancellationTokenSource.token); + + const events = spyingTelemetryService.getEvents(); + const toolsEvents = events.telemetryServiceEvents.filter( + e => e.eventName === 'request.options.tools' + ); + + expect(toolsEvents.length).toBe(1); + const props = toolsEvents[0]!.properties as Record; + expect(props.headerRequestId).toBeDefined(); + expect(props.messagesJson).toBeDefined(); + + const toolsPayload = JSON.parse(props.messagesJson); + expect(toolsPayload.length).toBe(1); + expect(toolsPayload[0].function.name).toBe('get_weather'); + }); + + it('does not emit request.options.tools event when no tools are present', async () => { + const endpointWithoutTools = createChatCompletionEndpointWithoutTools(); + mockFetcherService.queueResponse(createSuccessResponse('Hello!')); + + const opts: IFetchMLOptions = { + debugName: 'test-no-tools-telemetry', + messages: [{ role: Raw.ChatRole.User, content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'Hello' }] }], + endpoint: endpointWithoutTools, + location: ChatLocation.Panel, + requestOptions: {}, + finishedCb: undefined, + }; + + await fetcher.fetchMany(opts, cancellationTokenSource.token); + + const events = spyingTelemetryService.getEvents(); + const toolsEvents = events.telemetryServiceEvents.filter( + e => e.eventName === 'request.options.tools' + ); + + expect(toolsEvents.length).toBe(0); + }); + + it('multiplexes messagesJson when tool schemas exceed 8KB', async () => { + const endpointWithLargeTools = createEndpointWithLargeTools(); + mockFetcherService.queueResponse(createSuccessResponse('Hello!')); + + const opts: IFetchMLOptions = { + debugName: 'test-large-tools-telemetry', + messages: [{ role: Raw.ChatRole.User, content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'Use a tool' }] }], + endpoint: endpointWithLargeTools, + location: ChatLocation.Panel, + requestOptions: {}, + finishedCb: undefined, + }; + + await fetcher.fetchMany(opts, cancellationTokenSource.token); + + const events = spyingTelemetryService.getEvents(); + const toolsEvents = events.telemetryServiceEvents.filter( + e => e.eventName === 'request.options.tools' + ); + + expect(toolsEvents.length).toBe(1); + const props = toolsEvents[0]!.properties as Record; + + // The messagesJson value exceeds 8192 chars, so multiplexProperties should chunk it + expect(props.messagesJson).toBeDefined(); + expect(props.messagesJson_02).toBeDefined(); + + // Reassemble the chunked value and verify it's valid JSON + let fullJson = props.messagesJson; + let i = 2; + while (props[`messagesJson_${String(i).padStart(2, '0')}`]) { + fullJson += props[`messagesJson_${String(i).padStart(2, '0')}`]; + i++; + } + const toolsPayload = JSON.parse(fullJson); + expect(toolsPayload.length).toBeGreaterThan(0); + expect(toolsPayload[0].function.name).toBe('large_tool_0'); + }); +}); + // --- Test Helpers --- +function createEndpointWithTools(): IChatEndpoint { + return { + url: 'https://api.github.com/copilot/chat/completions', + urlOrRequestMetadata: 'https://api.github.com/copilot/chat/completions', + model: 'test-model', + modelMaxPromptTokens: 8192, + maxOutputTokens: 4096, + supportsToolCalls: true, + supportsVision: false, + supportsPrediction: false, + showInModelPicker: true, + isDefault: true, + isFallback: false, + policy: 'enabled', + getHeaders: async () => ({}), + createRequestBody: (): IEndpointBody => ({ + model: 'test-model', + messages: [{ role: 'user', content: 'Use a tool' }], + stream: true, + tools: [{ + type: 'function', + function: { + name: 'get_weather', + description: 'Get the weather for a location', + parameters: { type: 'object', properties: { location: { type: 'string' } } }, + }, + }], + }), + acquireTokenizer: () => ({ + countMessagesTokens: async () => 100, + countTokens: async () => 100, + tokenize: async () => [], + }), + processResponseFromChatEndpoint: async (_telemetryService: ITelemetryService, _logService: ILogService, response: Response, _expectedNumChoices: number, finishedCb: FinishedCallback, telemetryData: TelemetryData, _cancellationToken?: CancellationToken) => { + const text = await response.text(); + if (finishedCb) { + await finishedCb(text, 0, { text }); + } + return { + [Symbol.asyncIterator]: async function* () { + yield { + message: { role: Raw.ChatRole.Assistant, content: [{ type: Raw.ChatCompletionContentPartKind.Text, text }] }, + choiceIndex: 0, + requestId: { + headerRequestId: response.headers.get('x-request-id') || 'test-request-id', + gitHubRequestId: response.headers.get('x-github-request-id') || '', + completionId: '', + created: 0, + serverExperiments: '', + deploymentId: '', + }, + tokens: [], + usage: undefined, + model: 'test-model', + blockFinished: true, + finishReason: 'stop', + telemetryData: telemetryData, + }; + } + }; + }, + acceptChatPolicy: async () => true, + doRequest: async () => { + throw new Error('Not implemented'); + }, + } as unknown as IChatEndpoint; +} + +function createChatCompletionEndpointWithoutTools(): IChatEndpoint { + return { + url: 'https://api.github.com/copilot/chat/completions', + urlOrRequestMetadata: 'https://api.github.com/copilot/chat/completions', + model: 'test-model', + modelMaxPromptTokens: 8192, + maxOutputTokens: 4096, + supportsToolCalls: true, + supportsVision: false, + supportsPrediction: false, + showInModelPicker: true, + isDefault: true, + isFallback: false, + policy: 'enabled', + getHeaders: async () => ({}), + createRequestBody: (): IEndpointBody => ({ + model: 'test-model', + messages: [{ role: 'user', content: 'Hello' }], + stream: true, + // No tools field + }), + acquireTokenizer: () => ({ + countMessagesTokens: async () => 100, + countTokens: async () => 100, + tokenize: async () => [], + }), + processResponseFromChatEndpoint: async (_telemetryService: ITelemetryService, _logService: ILogService, response: Response, _expectedNumChoices: number, finishedCb: FinishedCallback, telemetryData: TelemetryData, _cancellationToken?: CancellationToken) => { + const text = await response.text(); + if (finishedCb) { + await finishedCb(text, 0, { text }); + } + return { + [Symbol.asyncIterator]: async function* () { + yield { + message: { role: Raw.ChatRole.Assistant, content: [{ type: Raw.ChatCompletionContentPartKind.Text, text }] }, + choiceIndex: 0, + requestId: { + headerRequestId: response.headers.get('x-request-id') || 'test-request-id', + gitHubRequestId: response.headers.get('x-github-request-id') || '', + completionId: '', + created: 0, + serverExperiments: '', + deploymentId: '', + }, + tokens: [], + usage: undefined, + model: 'test-model', + blockFinished: true, + finishReason: 'stop', + telemetryData: telemetryData, + }; + } + }; + }, + acceptChatPolicy: async () => true, + doRequest: async () => { + throw new Error('Not implemented'); + }, + } as unknown as IChatEndpoint; +} + +function createEndpointWithLargeTools(): IChatEndpoint { + // Generate tools with schemas large enough to exceed 8192 chars when JSON.stringified + const largeTools = Array.from({ length: 20 }, (_, i) => ({ + type: 'function' as const, + function: { + name: `large_tool_${i}`, + description: 'A'.repeat(500), + parameters: { + type: 'object', + properties: Object.fromEntries( + Array.from({ length: 10 }, (_, j) => [`param_${j}`, { type: 'string', description: 'B'.repeat(50) }]) + ), + }, + }, + })); + + return { + url: 'https://api.github.com/copilot/chat/completions', + urlOrRequestMetadata: 'https://api.github.com/copilot/chat/completions', + model: 'test-model', + modelMaxPromptTokens: 8192, + maxOutputTokens: 4096, + supportsToolCalls: true, + supportsVision: false, + supportsPrediction: false, + showInModelPicker: true, + isDefault: true, + isFallback: false, + policy: 'enabled', + getHeaders: async () => ({}), + createRequestBody: (): IEndpointBody => ({ + model: 'test-model', + messages: [{ role: 'user', content: 'Use a tool' }], + stream: true, + tools: largeTools, + }), + acquireTokenizer: () => ({ + countMessagesTokens: async () => 100, + countTokens: async () => 100, + tokenize: async () => [], + }), + processResponseFromChatEndpoint: async (_telemetryService: ITelemetryService, _logService: ILogService, response: Response, _expectedNumChoices: number, finishedCb: FinishedCallback, telemetryData: TelemetryData, _cancellationToken?: CancellationToken) => { + const text = await response.text(); + if (finishedCb) { + await finishedCb(text, 0, { text }); + } + return { + [Symbol.asyncIterator]: async function* () { + yield { + message: { role: Raw.ChatRole.Assistant, content: [{ type: Raw.ChatCompletionContentPartKind.Text, text }] }, + choiceIndex: 0, + requestId: { + headerRequestId: response.headers.get('x-request-id') || 'test-request-id', + gitHubRequestId: response.headers.get('x-github-request-id') || '', + completionId: '', + created: 0, + serverExperiments: '', + deploymentId: '', + }, + tokens: [], + usage: undefined, + model: 'test-model', + blockFinished: true, + finishReason: 'stop', + telemetryData: telemetryData, + }; + } + }; + }, + acceptChatPolicy: async () => true, + doRequest: async () => { + throw new Error('Not implemented'); + }, + } as unknown as IChatEndpoint; +} + /** * Creates an endpoint that returns Response API format request body (with input instead of messages) */