From 2409e1b3023b5b04731dd57040bc0f98ca449b3c Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 13:15:25 -0700 Subject: [PATCH 01/53] Add PackageVersion for Kusto.Data and Kusto.Ingest --- Directory.Packages.props | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 132c759586..b6bb1ed400 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -60,7 +60,8 @@ - + + From 6ed17648795caa95612549d62a5c3230a0e983e5 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 13:15:57 -0700 Subject: [PATCH 02/53] Add Id to shared McpModels. --- eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs b/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs index ad6ce6c59f..b4a68bb118 100644 --- a/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs +++ b/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs @@ -81,8 +81,11 @@ public class ToolAnnotations } // Tool definition for azmcp tools list response -public class Tool -{ +public class Tool { + + [JsonPropertyName("id")] + public string? Id { get; set; } + [JsonPropertyName("name")] public required string Name { get; set; } From 9e969b4e0268d620e57b05ee7c4bf171dd9dcecc Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 13:16:21 -0700 Subject: [PATCH 03/53] Initial commit --- .../ToolMetadataExporter/AppConfiguration.cs | 17 ++ .../ToolMetadataExporter/GlobalUsings.cs | 5 + .../Models/AzureMcpTool.cs | 6 + .../Models/Kusto/McpToolEvent.cs | 56 ++++ .../Models/Kusto/McpToolEventType.cs | 11 + .../Models/ModelsSerializationContext.cs | 14 + eng/tools/ToolMetadataExporter/Program.cs | 111 ++++++++ .../Resources/queries/GetAvailableTools.kql | 4 + .../Services/AzmcpProgram.cs | 19 ++ .../Services/AzureMcpKustoDatastore.cs | 177 ++++++++++++ .../Services/IAzureMcpDatastore.cs | 14 + .../ToolMetadataExporter.csproj | 32 +++ eng/tools/ToolMetadataExporter/Utility.cs | 254 ++++++++++++++++++ .../ToolMetadataExporter/appsettings.json | 5 + 14 files changed, 725 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/AppConfiguration.cs create mode 100644 eng/tools/ToolMetadataExporter/GlobalUsings.cs create mode 100644 eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs create mode 100644 eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs create mode 100644 eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs create mode 100644 eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs create mode 100644 eng/tools/ToolMetadataExporter/Program.cs create mode 100644 eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql create mode 100644 eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs create mode 100644 eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs create mode 100644 eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs create mode 100644 eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj create mode 100644 eng/tools/ToolMetadataExporter/Utility.cs create mode 100644 eng/tools/ToolMetadataExporter/appsettings.json diff --git a/eng/tools/ToolMetadataExporter/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/AppConfiguration.cs new file mode 100644 index 0000000000..3e9142a5d0 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/AppConfiguration.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter; + +public class AppConfiguration +{ + public string? ClusterName { get; set; } + + public string? DatabaseName { get; set; } + + public string? McpToolEventsTableName { get; set; } + + public string? QueriesFolder { get; set; } + + public string? WorkDirectory { get; set; } +} diff --git a/eng/tools/ToolMetadataExporter/GlobalUsings.cs b/eng/tools/ToolMetadataExporter/GlobalUsings.cs new file mode 100644 index 0000000000..38bac6b955 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/GlobalUsings.cs @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +global using System; +global using System.Text.Json; diff --git a/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs b/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs new file mode 100644 index 0000000000..60cc77c698 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs @@ -0,0 +1,6 @@ +namespace ToolMetadataExporter.Models; + +public record AzureMcpTool( + string ToolId, + string ToolName, + string ToolArea); diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs new file mode 100644 index 0000000000..0adc12a18e --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using Kusto.Data.Common; + +namespace ToolMetadataExporter.Models.Kusto; + +public class McpToolEvent +{ + private const string EventTimeColumn = "EventTime"; + private const string EventTypeColumn = "EventType"; + private const string ServerVersionColumn = "ServerVersion"; + private const string ToolIdColumn = "ToolId"; + private const string ToolNameColumn = "ToolName"; + private const string ToolAreaColumn = "ToolArea"; + private const string ReplacedByToolNameColumn = "ReplacedByToolName"; + private const string ReplacedByToolAreaColumn = "ReplacedByToolArea"; + + [JsonPropertyName(EventTimeColumn)] + public DateTimeOffset? EventTime { get; set; } + + [JsonPropertyName(EventTypeColumn)] + public McpToolEventType? EventType { get; set; } + + [JsonPropertyName(ServerVersionColumn)] + public string? ServerVersion { get; set; } + + [JsonPropertyName(ToolIdColumn)] + public string? ToolId { get; set; } + + [JsonPropertyName(ToolNameColumn)] + public string? ToolName { get; set; } + + [JsonPropertyName(ToolAreaColumn)] + public string? ToolArea { get; set; } + + [JsonPropertyName(ReplacedByToolNameColumn)] + public string? ReplacedByToolName { get; set; } + + [JsonPropertyName(ReplacedByToolAreaColumn)] + public string? ReplacedByToolArea { get; set; } + + public static ColumnMapping[] GetColumnMappings() + { + return [ + new ColumnMapping { ColumnName = EventTypeColumn, ColumnType = "datetime" }, + new ColumnMapping { ColumnName = ServerVersionColumn, ColumnType = "string" }, + new ColumnMapping { ColumnName = ToolIdColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ToolNameColumn, ColumnType = "string" }, + new ColumnMapping { ColumnName = ToolAreaColumn , ColumnType = "string" }, + new ColumnMapping { ColumnName = ReplacedByToolNameColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ReplacedByToolAreaColumn, ColumnType = "string"}, + ]; + } +} diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs new file mode 100644 index 0000000000..3c595bbc1b --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter.Models.Kusto; + +public enum McpToolEventType +{ + Created, + Updated, + Deleted +} diff --git a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs new file mode 100644 index 0000000000..e2b4805f98 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using ToolMetadataExporter.Models.Kusto; + +namespace ToolMetadataExporter.Models; + +[JsonSerializable(typeof(McpToolEvent))] +[JsonSerializable(typeof(McpToolEventType))] +[JsonSerializable(typeof(List))] +public partial class ModelsSerializationContext : JsonSerializerContext +{ +} diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs new file mode 100644 index 0000000000..8a6261d112 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ToolMetadataExporter.Models.Kusto; +using ToolMetadataExporter.Services; + +namespace ToolMetadataExporter; + +public class Program +{ + private readonly AzmcpProgram _azmcpExe; + private readonly IAzureMcpDatastore _azureMcpDatastore; + private readonly ILogger _logger; + + public Program(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, ILogger logger) + { + _azmcpExe = program; + _azureMcpDatastore = azureMcpDatastore; + _logger = logger; + } + + public static async Task Main(string[] args) + { + var builder = Host.CreateApplicationBuilder(args); + ConfigureServices(builder.Services); + + var host = builder.Build(); + + var program = host.Services.GetRequiredService(); + + await program.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); + await host.RunAsync(); + } + + private static void ConfigureServices(IServiceCollection services) + { + + services.AddSingleton() + .AddSingleton(); + + services.AddOptions() + .PostConfigure(existing => + { + if (existing.WorkDirectory == null) + { + string exeDir = AppContext.BaseDirectory; + var repoRoot = Utility.FindRepoRoot(exeDir); + existing.WorkDirectory = Path.Combine(repoRoot, ".work"); + } + }); + + services.AddLogging(builder => + { + builder.AddConsole(); + }); + } + + private async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, CancellationToken cancellationToken = default) + { + _logger.LogInformation("Starting analysis."); + + var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); + + if (currentTools == null) + { + _logger.LogError("LoadToolsDynamicallyAsync did not return a result."); + return; + } + else if (currentTools.Tools == null || currentTools.Tools.Count == 0) + { + _logger.LogWarning("azmcp program did not return any tools."); + return; + } + + var existingTools = (await _azureMcpDatastore.GetAvailableToolsAsync(cancellationToken)).ToDictionary(x => x.ToolId); + + if (cancellationToken.IsCancellationRequested) + { + _logger.LogInformation("Analysis was cancelled."); + return; + } + + var changes = new List(); + + foreach (var tool in currentTools.Tools) + { + if (string.IsNullOrEmpty(tool.Id)) + { + throw new InvalidOperationException($"Tool without an id. Name: {tool.Name}. Command: {tool.Command}"); + } + + if (!existingTools.TryGetValue(tool.Id, out var knownValue)) + { + + } + else + { + changes.Add(new McpToolEvent + { + EventTime = analysisTime, + EventType = McpToolEventType.Created, + }); + } + } + + await _azureMcpDatastore.AddToolEventsAsync(changes, cancellationToken); + } +} diff --git a/eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql b/eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql new file mode 100644 index 0000000000..26a901adc8 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql @@ -0,0 +1,4 @@ +McpToolEvents + | summarize arg_max(EventTime, *) by ToolId + | where EventType != 'Deleted' + | project EventTime, ToolId, ToolName, ToolArea, ServerVersion, EventType, ReplacedByToolName, ReplacedByToolArea diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs new file mode 100644 index 0000000000..f58b0558e2 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.Options; +using ToolSelection.Models; + +namespace ToolMetadataExporter.Services; + +public class AzmcpProgram +{ + private readonly string _toolDirectory; + + public AzmcpProgram(IOptions options) + { + _toolDirectory = options.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); + } + + public virtual Task LoadToolsDynamicallyAsync() + { + return Utility.LoadToolsDynamicallyAsync(_toolDirectory, false); + } +} diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs new file mode 100644 index 0000000000..beb756234a --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.CompilerServices; +using Kusto.Data.Common; +using Kusto.Data.Ingestion; +using Kusto.Ingest; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; + +namespace ToolMetadataExporter.Services; + +public class AzureMcpKustoDatastore : IAzureMcpDatastore +{ + private readonly ICslQueryProvider _kustoClient; + private readonly IKustoIngestClient _ingestClient; + private readonly ILogger _logger; + private readonly DirectoryInfo _queriesDirectory; + private readonly string _databaseName; + private readonly string _tableName; + + public AzureMcpKustoDatastore( + ICslQueryProvider kustoClient, + IKustoIngestClient ingestClient, + IOptions configuration, + ILogger logger) + { + _kustoClient = kustoClient; + _ingestClient = ingestClient; + _logger = logger; + + _databaseName = configuration.Value.DatabaseName ?? throw new ArgumentNullException(nameof(AppConfiguration.DatabaseName)); + _tableName = configuration.Value.McpToolEventsTableName ?? throw new ArgumentNullException(nameof(AppConfiguration.McpToolEventsTableName)); + _queriesDirectory = configuration.Value.QueriesFolder == null + ? throw new ArgumentNullException(nameof(configuration.Value.QueriesFolder)) + : new DirectoryInfo(configuration.Value.QueriesFolder); + + if (!_queriesDirectory.Exists) + { + throw new ArgumentException($"'{_queriesDirectory.FullName}' does not exist. Value: {configuration.Value.QueriesFolder}"); + } + } + + public async Task> GetAvailableToolsAsync(CancellationToken cancellationToken = default) + { + var queryFile = _queriesDirectory.GetFiles("GetAvailableTools.kql").FirstOrDefault(); + + if (queryFile == null) + { + throw new InvalidOperationException($"Could not find GetAvailableTools.kql in {_queriesDirectory.FullName}"); + } + + var results = new List(); + + await foreach (var latestEvent in GetLatestToolEventsAsync(queryFile.FullName, cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrEmpty(latestEvent.ToolId)) + { + throw new InvalidOperationException( + $"Cannot have an event with no id. Name: {latestEvent.ToolArea}, Area: {latestEvent.ToolArea}"); + } + + string? toolName; + string? toolArea; + switch (latestEvent.EventType) + { + case McpToolEventType.Created: + toolName = latestEvent.ToolName; + toolArea = latestEvent.ToolArea; + break; + case McpToolEventType.Updated: + toolName = latestEvent.ReplacedByToolName; + toolArea = latestEvent.ReplacedByToolArea; + break; + default: + throw new InvalidOperationException($"Tool '{latestEvent.ToolId}' has unsupported event type: {latestEvent.EventType}"); + } + + if (string.IsNullOrEmpty(toolName) || string.IsNullOrEmpty(toolArea)) + { + throw new InvalidOperationException($"Tool '{latestEvent.ToolId}' without tool name and/or a tool area."); + } + + results.Add(new AzureMcpTool(latestEvent.ToolId, toolName, toolArea)); + } + + return results; + } + + public async Task AddToolEventsAsync(IList toolEvents, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + using MemoryStream stream = new MemoryStream(); + + await JsonSerializer.SerializeAsync(stream, toolEvents, ModelsSerializationContext.Default.ListMcpToolEvent, cancellationToken); + stream.Seek(0, SeekOrigin.Begin); + + cancellationToken.ThrowIfCancellationRequested(); + + var ingestionProperties = new KustoIngestionProperties(_databaseName, _tableName) + { + Format = DataSourceFormat.json, + IngestionMapping = new IngestionMapping() + { + IngestionMappingKind = IngestionMappingKind.Json, + IngestionMappings = McpToolEvent.GetColumnMappings() + } + }; + + await _ingestClient.IngestFromStreamAsync(stream, ingestionProperties); + } + + internal async IAsyncEnumerable GetLatestToolEventsAsync(string kqlFilePath, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (!File.Exists(kqlFilePath)) + { + throw new FileNotFoundException($"KQL file not found: {kqlFilePath}"); + } + + var kql = await File.ReadAllTextAsync(kqlFilePath); + + var clientRequestProperties = new ClientRequestProperties(); + var reader = await _kustoClient.ExecuteQueryAsync(_databaseName, kql, clientRequestProperties, cancellationToken); + + var eventTimeOrdinal = reader.GetOrdinal("EventTime"); + var eventTypeOrdinal = reader.GetOrdinal("EventType"); + var serverVersionOrdinal = reader.GetOrdinal("ServerVersion"); + var toolIdOrdinal = reader.GetOrdinal("ToolId"); + var toolNameOrdinal = reader.GetOrdinal("ToolName"); + var toolAreaOrdinal = reader.GetOrdinal("ToolArea"); + var replacedByToolNameOrdinal = reader.GetOrdinal("ReplacedByToolName"); + var replacedByToolAreaOrdinal = reader.GetOrdinal("ReplacedByToolArea"); + + while (reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + + var eventTime = reader.GetDateTime(eventTimeOrdinal); + var eventTypeString = reader.GetString(eventTypeOrdinal); + var serverVersion = reader.GetString(serverVersionOrdinal); + var toolId = reader.GetString(toolIdOrdinal); + var toolName = reader.GetString(toolNameOrdinal); + var toolArea = reader.GetString(toolAreaOrdinal); + var replacedByToolName = reader.IsDBNull(replacedByToolNameOrdinal) + ? null + : reader.GetString(replacedByToolNameOrdinal); + var replacedByToolArea = reader.IsDBNull(replacedByToolAreaOrdinal) + ? null + : reader.GetString(replacedByToolAreaOrdinal); + + if (!Enum.TryParse(eventTypeString, ignoreCase: true, out var eventType)) + { + throw new InvalidOperationException($"Invalid EventType value: '{eventTypeString}'. EventTime: '{eventTime}', ToolName: '{toolName}', ToolArea: '{toolArea}'"); + } + + var tool = new McpToolEvent + { + EventTime = eventTime, + EventType = eventType, + ServerVersion = serverVersion, + ToolId = toolId, + ToolName = toolName, + ToolArea = toolArea, + ReplacedByToolName = replacedByToolName, + ReplacedByToolArea = replacedByToolArea, + }; + + yield return tool; + } + } +} diff --git a/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs b/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs new file mode 100644 index 0000000000..ace467be80 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; + +namespace ToolMetadataExporter.Services; + +public interface IAzureMcpDatastore +{ + Task> GetAvailableToolsAsync(CancellationToken cancellationToken = default); + + Task AddToolEventsAsync(IList toolEvents, CancellationToken cancellationToken = default); +} diff --git a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj new file mode 100644 index 0000000000..66f7146ca9 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj @@ -0,0 +1,32 @@ + + + + Exe + net9.0 + enable + enable + true + true + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs new file mode 100644 index 0000000000..bc2acabd75 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -0,0 +1,254 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Text.Json; +using ToolSelection.Models; + +namespace ToolMetadataExporter; + +internal class Utility +{ + internal static async Task LoadToolsDynamicallyAsync(string workDirectory, bool isCiMode = false) + { + try + { + // Locate azmcp artifact across common build outputs (servers/core, Debug/Release) + var exeDir = AppContext.BaseDirectory; + var repoRoot = FindRepoRoot(exeDir); + var searchRoots = new List + { + Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug"), + Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Release") + }; + + var candidateNames = new[] { "azmcp.exe", "azmcp", "azmcp.dll" }; + FileInfo? cliArtifact = null; + + foreach (var root in searchRoots.Where(Directory.Exists)) + { + foreach (var name in candidateNames) + { + var found = new DirectoryInfo(root) + .EnumerateFiles(name, SearchOption.AllDirectories) + .FirstOrDefault(); + if (found != null) + { + cliArtifact = found; + break; + } + } + + if (cliArtifact != null) + { + break; + } + } + + if (cliArtifact == null) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); + } + + var isDll = string.Equals(cliArtifact.Extension, ".dll", StringComparison.OrdinalIgnoreCase); + var fileName = isDll ? "dotnet" : cliArtifact.FullName; + var arguments = isDll ? $"{cliArtifact.FullName} tools list" : "tools list"; + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + } + }; + + process.Start(); + + var output = await process.StandardOutput.ReadToEndAsync(); + var error = await process.StandardError.ReadToEndAsync(); + + await process.WaitForExitAsync(); + + if (process.ExitCode != 0) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw new InvalidOperationException($"Failed to get tools from azmcp: {error}"); + } + + // Filter out non-JSON lines (like launch settings messages) + var lines = output.Split('\n'); + var jsonStartIndex = -1; + + for (int i = 0; i < lines.Length; i++) + { + if (lines[i].Trim().StartsWith("{")) + { + jsonStartIndex = i; + + break; + } + } + + if (jsonStartIndex == -1) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw new InvalidOperationException("No JSON output found from azmcp command."); + } + + var jsonOutput = string.Join('\n', lines.Skip(jsonStartIndex)); + + // Parse the JSON output + var result = JsonSerializer.Deserialize(jsonOutput, SourceGenerationContext.Default.ListToolsResult); + + // Save the dynamically loaded tools to tools.json for future use + if (result != null) + { + await SaveToolsToJsonAsync(result, Path.Combine(workDirectory, "tools.json")); + + Console.WriteLine($"💾 Saved {result.Tools?.Count} tools to tools.json"); + } + + return result; + } + catch (Exception) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw; + } + } + + private static async Task LoadToolsFromJsonAsync(string filePath, bool isCiMode = false) + { + if (!File.Exists(filePath)) + { + if (isCiMode) + { + return null; // Let caller handle this gracefully + } + + throw new FileNotFoundException($"Tools file not found: {filePath}"); + } + + var json = await File.ReadAllTextAsync(filePath); + + // Process the JSON + if (json.StartsWith('\'') && json.EndsWith('\'')) + { + json = json[1..^1]; // Remove first and last characters (quotes) + json = json.Replace("\\'", "'"); // Convert \' --> ' + json = json.Replace("\\\\\"", "'"); // Convert \\" --> ' + } + + var result = JsonSerializer.Deserialize(json, SourceGenerationContext.Default.ListToolsResult); + + return result; + } + + private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, string filePath) + { + try + { + // Normalize only tool and option descriptions instead of escaping the entire JSON document + if (toolsResult.Tools != null) + { + foreach (var tool in toolsResult.Tools) + { + if (!string.IsNullOrEmpty(tool.Description)) + { + tool.Description = EscapeCharacters(tool.Description); + } + + if (tool.Options != null) + { + foreach (var opt in tool.Options) + { + if (!string.IsNullOrEmpty(opt.Description)) + { + opt.Description = EscapeCharacters(opt.Description); + } + } + } + } + } + + var writerOptions = new JsonWriterOptions + { + Indented = true, + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + using var stream = new MemoryStream(); + using (var jsonWriter = new Utf8JsonWriter(stream, writerOptions)) + { + JsonSerializer.Serialize(jsonWriter, toolsResult, SourceGenerationContext.Default.ListToolsResult); + } + + await File.WriteAllBytesAsync(filePath, stream.ToArray()); + } + catch (Exception ex) + { + Console.WriteLine($"⚠️ Warning: Failed to save tools to {filePath}: {ex.Message}"); + Console.WriteLine(ex.StackTrace); + } + } + + private static string EscapeCharacters(string text) + { + if (string.IsNullOrEmpty(text)) + return text; + + // Normalize only the fancy “curly” quotes to straight ASCII. Identity replacements were removed. + return text.Replace(UnicodeChars.LeftSingleQuote, "'") + .Replace(UnicodeChars.RightSingleQuote, "'") + .Replace(UnicodeChars.LeftDoubleQuote, "\"") + .Replace(UnicodeChars.RightDoubleQuote, "\""); + } + + // Traverse up from a starting directory to find the repo root (containing AzureMcp.sln or .git) + internal static string FindRepoRoot(string startDir) + { + var dir = new DirectoryInfo(startDir); + + while (dir != null) + { + if (File.Exists(Path.Combine(dir.FullName, "AzureMcp.sln")) || + Directory.Exists(Path.Combine(dir.FullName, ".git"))) + { + return dir.FullName; + } + dir = dir.Parent; + } + + throw new InvalidOperationException("Could not find repo root (AzureMcp.sln or .git)."); + } + + internal static class UnicodeChars + { + public const string LeftSingleQuote = "\u2018"; + public const string RightSingleQuote = "\u2019"; + public const string LeftDoubleQuote = "\u201C"; + public const string RightDoubleQuote = "\u201D"; + } +} diff --git a/eng/tools/ToolMetadataExporter/appsettings.json b/eng/tools/ToolMetadataExporter/appsettings.json new file mode 100644 index 0000000000..00edbc5dff --- /dev/null +++ b/eng/tools/ToolMetadataExporter/appsettings.json @@ -0,0 +1,5 @@ +{ + "ClusterName": "McpClusterName", + "DatabaseName": "McpDatastore", + "McpToolEventsTableName": "McpToolEvents" +} From 401a5c30c788122dea5b1eaa2a23a5ac95234248 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:02:19 -0700 Subject: [PATCH 04/53] Remove AoT --- eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj index 66f7146ca9..ed5ce8d082 100644 --- a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj +++ b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj @@ -5,7 +5,6 @@ net9.0 enable enable - true true From c408b27ab0efdb6fea5f1038b59e4cda5c4dce39 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:02:33 -0700 Subject: [PATCH 05/53] Add initial mock up --- .../ToolMetadataExporter/AppConfiguration.cs | 2 +- eng/tools/ToolMetadataExporter/Program.cs | 110 +++++------ .../Services/AzmcpProgram.cs | 5 + .../ToolMetadataExporter/ToolAnalyzer.cs | 172 ++++++++++++++++++ eng/tools/ToolMetadataExporter/Utility.cs | 164 +++++++++-------- .../ToolMetadataExporter/appsettings.json | 13 +- 6 files changed, 318 insertions(+), 148 deletions(-) create mode 100644 eng/tools/ToolMetadataExporter/ToolAnalyzer.cs diff --git a/eng/tools/ToolMetadataExporter/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/AppConfiguration.cs index 3e9142a5d0..7f7d8fbd6c 100644 --- a/eng/tools/ToolMetadataExporter/AppConfiguration.cs +++ b/eng/tools/ToolMetadataExporter/AppConfiguration.cs @@ -5,7 +5,7 @@ namespace ToolMetadataExporter; public class AppConfiguration { - public string? ClusterName { get; set; } + public string? ClusterEndpoint { get; set; } public string? DatabaseName { get; set; } diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 8a6261d112..8de4b203e5 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -1,48 +1,50 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Azure.Core; +using Azure.Identity; +using Kusto.Data; +using Kusto.Data.Common; +using Kusto.Data.Net.Client; +using Kusto.Ingest; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ToolMetadataExporter.Models.Kusto; +using Microsoft.Extensions.Options; using ToolMetadataExporter.Services; namespace ToolMetadataExporter; public class Program { - private readonly AzmcpProgram _azmcpExe; - private readonly IAzureMcpDatastore _azureMcpDatastore; - private readonly ILogger _logger; - - public Program(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, ILogger logger) - { - _azmcpExe = program; - _azureMcpDatastore = azureMcpDatastore; - _logger = logger; - } - public static async Task Main(string[] args) { var builder = Host.CreateApplicationBuilder(args); - ConfigureServices(builder.Services); + ConfigureServices(builder.Services, builder.Configuration); + ConfigureAzureServices(builder.Services); var host = builder.Build(); - var program = host.Services.GetRequiredService(); + var analyzer = host.Services.GetRequiredService(); - await program.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); + await analyzer.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); await host.RunAsync(); } - private static void ConfigureServices(IServiceCollection services) + private static void ConfigureServices(IServiceCollection services, IConfiguration configuration) { + services.AddLogging(builder => + { + builder.AddConsole(); + }); services.AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); - services.AddOptions() - .PostConfigure(existing => + services.Configure(configuration.GetSection("AppConfig")) + .PostConfigure(existing => { if (existing.WorkDirectory == null) { @@ -51,61 +53,33 @@ private static void ConfigureServices(IServiceCollection services) existing.WorkDirectory = Path.Combine(repoRoot, ".work"); } }); - - services.AddLogging(builder => - { - builder.AddConsole(); - }); } - private async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, CancellationToken cancellationToken = default) + private static void ConfigureAzureServices(IServiceCollection services) { - _logger.LogInformation("Starting analysis."); - - var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); + services.AddScoped(sp => { + var credential = new ChainedTokenCredential( + new ManagedIdentityCredential(), + new DefaultAzureCredential() + ); - if (currentTools == null) - { - _logger.LogError("LoadToolsDynamicallyAsync did not return a result."); - return; - } - else if (currentTools.Tools == null || currentTools.Tools.Count == 0) - { - _logger.LogWarning("azmcp program did not return any tools."); - return; - } - - var existingTools = (await _azureMcpDatastore.GetAvailableToolsAsync(cancellationToken)).ToDictionary(x => x.ToolId); - - if (cancellationToken.IsCancellationRequested) - { - _logger.LogInformation("Analysis was cancelled."); - return; - } - - var changes = new List(); - - foreach (var tool in currentTools.Tools) + return credential; + }); + services.AddSingleton(sp => { - if (string.IsNullOrEmpty(tool.Id)) - { - throw new InvalidOperationException($"Tool without an id. Name: {tool.Name}. Command: {tool.Command}"); - } + var config = sp.GetRequiredService>(); + var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.ClusterEndpoint) + .WithAadAzureTokenCredentialsAuthentication(credential); - if (!existingTools.TryGetValue(tool.Id, out var knownValue)) - { - - } - else - { - changes.Add(new McpToolEvent - { - EventTime = analysisTime, - EventType = McpToolEventType.Created, - }); - } - } + return KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder); + }); + services.AddSingleton(sp => { + var config = sp.GetRequiredService>(); + var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.ClusterEndpoint); - await _azureMcpDatastore.AddToolEventsAsync(changes, cancellationToken); + return KustoIngestFactory.CreateQueuedIngestClient(connectionStringBuilder); + }); } } diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs index f58b0558e2..6a645bb567 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -16,4 +16,9 @@ public AzmcpProgram(IOptions options) { return Utility.LoadToolsDynamicallyAsync(_toolDirectory, false); } + + public Task GetVersionAsync() + { + return Utility.GetVersionAsync(); + } } diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs new file mode 100644 index 0000000000..6e1b67a978 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -0,0 +1,172 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; +using ToolMetadataExporter.Services; +using ToolSelection.Models; + +namespace ToolMetadataExporter; + +public class ToolAnalyzer +{ + private readonly AzmcpProgram _azmcpExe; + private readonly IAzureMcpDatastore _azureMcpDatastore; + private readonly ILogger _logger; + private readonly string _workingDirectory; + + public ToolAnalyzer(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, + IOptions configuration, ILogger logger) + { + _azmcpExe = program; + _azureMcpDatastore = azureMcpDatastore; + _logger = logger; + + _workingDirectory = configuration.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); + ; + } + + public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, CancellationToken cancellationToken = default) + { + _logger.LogInformation("Starting analysis."); + + var serverVersion = await _azmcpExe.GetVersionAsync(); + var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); + + if (currentTools == null) + { + _logger.LogError("LoadToolsDynamicallyAsync did not return a result."); + return; + } + else if (currentTools.Tools == null || currentTools.Tools.Count == 0) + { + _logger.LogWarning("azmcp program did not return any tools."); + return; + } + + var existingTools = (await _azureMcpDatastore.GetAvailableToolsAsync(cancellationToken)).ToDictionary(x => x.ToolId); + + if (cancellationToken.IsCancellationRequested) + { + _logger.LogInformation("Analysis was cancelled."); + return; + } + + // Iterate through all the current tools and match them against the + // state Kusto knows about. + // For each tool, if there is no matching Tool, it is a new tool. + // Else, check the ToolName and ToolArea. If either of those are different + // then there is an update. + var changes = new List(); + + foreach (var tool in currentTools.Tools) + { + if (string.IsNullOrEmpty(tool.Id)) + { + throw new InvalidOperationException($"Tool without an id. Name: {tool.Name}. Command: {tool.Command}"); + } + + var toolArea = GetToolArea(tool); + if (string.IsNullOrEmpty(toolArea)) + { + throw new InvalidOperationException($"Tool without a tool area. Name: {tool.Name}. Id: {tool.Id}"); + } + + var changeEvent = new McpToolEvent + { + EventTime = analysisTime, + ToolId = tool.Id, + ServerVersion = serverVersion, + }; + + var hasChange = false; + if (existingTools.Remove(tool.Id, out var knownValue)) + { + if (!string.Equals(tool.Name, knownValue.ToolName, StringComparison.OrdinalIgnoreCase) + || !string.Equals(toolArea, knownValue.ToolArea, StringComparison.OrdinalIgnoreCase)) + { + hasChange = true; + + changeEvent.EventType = McpToolEventType.Updated; + changeEvent.ToolName = knownValue.ToolName; + changeEvent.ToolArea = knownValue.ToolArea; + changeEvent.ReplacedByToolName = tool.Name; + changeEvent.ReplacedByToolArea = toolArea; + } + } + else + { + hasChange = true; + changeEvent.EventType = McpToolEventType.Created; + changeEvent.ToolName = tool.Name; + changeEvent.ToolArea = toolArea; + } + + if (hasChange) + { + changes.Add(changeEvent); + } + } + + // We're done iterating through the newest available tools. + // Any remaining entries in `existingTool` are ones that got deleted. + var removals = existingTools.Select(x => new McpToolEvent + { + ServerVersion = serverVersion, + EventTime = analysisTime, + EventType = McpToolEventType.Deleted, + ToolId = x.Key, + ToolName = x.Value.ToolName, + ToolArea = x.Value.ToolArea, + ReplacedByToolName = null, + ReplacedByToolArea = null, + }); + + changes.AddRange(removals); + + cancellationToken.ThrowIfCancellationRequested(); + + if (!changes.Any()) + { + _logger.LogInformation("No changes made."); + return; + } + + var outputFile = Path.Combine(_workingDirectory, "tool_changes.json"); + + _logger.LogInformation("Tool updates. Writing output to : {FileName}", outputFile); + + + var writerOptions = new JsonWriterOptions + { + Indented = true, + }; + + using (var ms = new MemoryStream()) + using (var jsonWriter = new Utf8JsonWriter(ms, writerOptions)) + { + JsonSerializer.Serialize(jsonWriter, changes, ModelsSerializationContext.Default.ListMcpToolEvent); + + try + { + await File.WriteAllBytesAsync(outputFile, ms.ToArray()); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error writing to {FileName}", outputFile); + } + } + + if (!isDryRun) + { + _logger.LogInformation("Updating datastore."); + await _azureMcpDatastore.AddToolEventsAsync(changes, cancellationToken); + } + } + + private static string? GetToolArea(Tool tool) + { + var split = tool.Command?.Split(" ", 2); + return split == null ? null : split[0]; + } + +} diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs index bc2acabd75..a7c027b0e5 100644 --- a/eng/tools/ToolMetadataExporter/Utility.cs +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -13,82 +13,7 @@ internal class Utility { try { - // Locate azmcp artifact across common build outputs (servers/core, Debug/Release) - var exeDir = AppContext.BaseDirectory; - var repoRoot = FindRepoRoot(exeDir); - var searchRoots = new List - { - Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug"), - Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Release") - }; - - var candidateNames = new[] { "azmcp.exe", "azmcp", "azmcp.dll" }; - FileInfo? cliArtifact = null; - - foreach (var root in searchRoots.Where(Directory.Exists)) - { - foreach (var name in candidateNames) - { - var found = new DirectoryInfo(root) - .EnumerateFiles(name, SearchOption.AllDirectories) - .FirstOrDefault(); - if (found != null) - { - cliArtifact = found; - break; - } - } - - if (cliArtifact != null) - { - break; - } - } - - if (cliArtifact == null) - { - if (isCiMode) - { - return null; // Graceful fallback in CI - } - - throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); - } - - var isDll = string.Equals(cliArtifact.Extension, ".dll", StringComparison.OrdinalIgnoreCase); - var fileName = isDll ? "dotnet" : cliArtifact.FullName; - var arguments = isDll ? $"{cliArtifact.FullName} tools list" : "tools list"; - - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = fileName, - Arguments = arguments, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true - } - }; - - process.Start(); - - var output = await process.StandardOutput.ReadToEndAsync(); - var error = await process.StandardError.ReadToEndAsync(); - - await process.WaitForExitAsync(); - - if (process.ExitCode != 0) - { - if (isCiMode) - { - return null; // Graceful fallback in CI - } - - throw new InvalidOperationException($"Failed to get tools from azmcp: {error}"); - } - + var output = await ExecuteAzmcpAsync("tools list", isCiMode); // Filter out non-JSON lines (like launch settings messages) var lines = output.Split('\n'); var jsonStartIndex = -1; @@ -139,6 +64,92 @@ internal class Utility } } + internal static async Task GetVersionAsync() + { + return await ExecuteAzmcpAsync("version"); + } + + internal static async Task ExecuteAzmcpAsync(string arguments, bool isCiMode = false) + { + // Locate azmcp artifact across common build outputs (servers/core, Debug/Release) + var exeDir = AppContext.BaseDirectory; + var repoRoot = FindRepoRoot(exeDir); + var searchRoots = new List + { + Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug"), + Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Release") + }; + + var candidateNames = new[] { "azmcp.exe", "azmcp", "azmcp.dll" }; + FileInfo? cliArtifact = null; + + foreach (var root in searchRoots.Where(Directory.Exists)) + { + foreach (var name in candidateNames) + { + var found = new DirectoryInfo(root) + .EnumerateFiles(name, SearchOption.AllDirectories) + .FirstOrDefault(); + if (found != null) + { + cliArtifact = found; + break; + } + } + + if (cliArtifact != null) + { + break; + } + } + + if (cliArtifact == null) + { + if (isCiMode) + { + return string.Empty; // Graceful fallback in CI + } + + throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); + } + + var isDll = string.Equals(cliArtifact.Extension, ".dll", StringComparison.OrdinalIgnoreCase); + var fileName = isDll ? "dotnet" : cliArtifact.FullName; + var argumentsToUse = isDll ? $"{cliArtifact.FullName} " : "tools list"; + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = argumentsToUse, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + } + }; + + process.Start(); + + var output = await process.StandardOutput.ReadToEndAsync(); + var error = await process.StandardError.ReadToEndAsync(); + + await process.WaitForExitAsync(); + + if (process.ExitCode != 0) + { + if (isCiMode) + { + return string.Empty; // Graceful fallback in CI + } + + throw new InvalidOperationException($"Failed to execute operation '{arguments}' from azmcp: {error}"); + } + + return output; + } + private static async Task LoadToolsFromJsonAsync(string filePath, bool isCiMode = false) { if (!File.Exists(filePath)) @@ -166,6 +177,7 @@ internal class Utility return result; } + private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, string filePath) { try diff --git a/eng/tools/ToolMetadataExporter/appsettings.json b/eng/tools/ToolMetadataExporter/appsettings.json index 00edbc5dff..456c353a6b 100644 --- a/eng/tools/ToolMetadataExporter/appsettings.json +++ b/eng/tools/ToolMetadataExporter/appsettings.json @@ -1,5 +1,12 @@ { - "ClusterName": "McpClusterName", - "DatabaseName": "McpDatastore", - "McpToolEventsTableName": "McpToolEvents" + "Logging": { + "LogLevel": { + "Default": "Debug" + } + }, + "AppConfig": { + "ClusterName": "McpClusterName", + "DatabaseName": "McpDatastore", + "McpToolEventsTableName": "McpToolEvents" + } } From d457f28b48a2a31b9a9e94a527c690950ddb048e Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:06:49 -0700 Subject: [PATCH 06/53] Split to use Query and Ingestion endpoint. --- eng/tools/ToolMetadataExporter/AppConfiguration.cs | 4 +++- eng/tools/ToolMetadataExporter/appsettings.json | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/AppConfiguration.cs index 7f7d8fbd6c..881679e435 100644 --- a/eng/tools/ToolMetadataExporter/AppConfiguration.cs +++ b/eng/tools/ToolMetadataExporter/AppConfiguration.cs @@ -5,7 +5,9 @@ namespace ToolMetadataExporter; public class AppConfiguration { - public string? ClusterEndpoint { get; set; } + public string? IngestionEndpoint { get; set; } + + public string? QueryEndpoint { get; set; } public string? DatabaseName { get; set; } diff --git a/eng/tools/ToolMetadataExporter/appsettings.json b/eng/tools/ToolMetadataExporter/appsettings.json index 456c353a6b..fd78b76088 100644 --- a/eng/tools/ToolMetadataExporter/appsettings.json +++ b/eng/tools/ToolMetadataExporter/appsettings.json @@ -5,7 +5,8 @@ } }, "AppConfig": { - "ClusterName": "McpClusterName", + "IngestionEndpoint": "https://ingest-your-server.region.kusto.windows.net", + "QueryEndpoint": "https://your-server.region.kusto.windows.net", "DatabaseName": "McpDatastore", "McpToolEventsTableName": "McpToolEvents" } From 1f9aa3c992d3916d26e6ae0d5a960b09c01f8685 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:17:30 -0700 Subject: [PATCH 07/53] Add Query to create table. --- .../Resources/queries/CreateTable.kql | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql diff --git a/eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql b/eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql new file mode 100644 index 0000000000..74a14c1c0d --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql @@ -0,0 +1,10 @@ +.create table McpToolEvents ( + EventTime: datetime, + EventType: string, + ServerVersion: string, + ToolId: string, + ToolName: string, + ToolArea: string, + ReplacedByToolName: string, + ReplacedByToolArea: string +) From 28218376b5e6a619230741979bee5992490aced9 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:38:44 -0700 Subject: [PATCH 08/53] Add LaunchSettings. --- .../Properties/launchSettings.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/Properties/launchSettings.json diff --git a/eng/tools/ToolMetadataExporter/Properties/launchSettings.json b/eng/tools/ToolMetadataExporter/Properties/launchSettings.json new file mode 100644 index 0000000000..7f48f76e44 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/Properties/launchSettings.json @@ -0,0 +1,10 @@ +{ + "profiles": { + "ToolMetadataExporter": { + "commandName": "Project", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} From 489992e0adcb9d982a5ed9db0a01abe1489386f2 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 23 Oct 2025 15:38:57 -0700 Subject: [PATCH 09/53] Set Default queries folder. --- eng/tools/ToolMetadataExporter/AppConfiguration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/AppConfiguration.cs index 881679e435..915802ae62 100644 --- a/eng/tools/ToolMetadataExporter/AppConfiguration.cs +++ b/eng/tools/ToolMetadataExporter/AppConfiguration.cs @@ -13,7 +13,7 @@ public class AppConfiguration public string? McpToolEventsTableName { get; set; } - public string? QueriesFolder { get; set; } + public string? QueriesFolder { get; set; } = "Resources/queries"; public string? WorkDirectory { get; set; } } From ab173371752eee94a5dea52e0ba93ebd9cbc4cab Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 28 Oct 2025 09:40:17 -0700 Subject: [PATCH 10/53] Adding Appsettings.json. Copy to output --- .../ToolAnalyzerTests.cs | 8 ++++++++ eng/tools/ToolMetadataExporter/Program.cs | 12 +++++++----- .../ToolMetadataExporter/ToolMetadataExporter.csproj | 6 ++++++ eng/tools/ToolMetadataExporter/appsettings.json | 5 ----- 4 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs diff --git a/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs b/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs new file mode 100644 index 0000000000..4882f1233f --- /dev/null +++ b/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter.UnitTests; + +public class ToolAnalyzerTests +{ +} diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 8de4b203e5..2ff963d969 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -21,6 +21,7 @@ public class Program public static async Task Main(string[] args) { var builder = Host.CreateApplicationBuilder(args); + ConfigureServices(builder.Services, builder.Configuration); ConfigureAzureServices(builder.Services); @@ -68,16 +69,17 @@ private static void ConfigureAzureServices(IServiceCollection services) services.AddSingleton(sp => { var config = sp.GetRequiredService>(); - var credential = sp.GetRequiredService(); - var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.ClusterEndpoint) - .WithAadAzureTokenCredentialsAuthentication(credential); + //var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.QueryEndpoint) + .WithAadAzCliAuthentication(interactive: true); return KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder); }); services.AddSingleton(sp => { var config = sp.GetRequiredService>(); - var credential = sp.GetRequiredService(); - var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.ClusterEndpoint); + //var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.IngestionEndpoint) + .WithAadAzCliAuthentication(interactive: true); return KustoIngestFactory.CreateQueuedIngestClient(connectionStringBuilder); }); diff --git a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj index ed5ce8d082..39d1ece9b1 100644 --- a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj +++ b/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj @@ -23,6 +23,12 @@ + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/eng/tools/ToolMetadataExporter/appsettings.json b/eng/tools/ToolMetadataExporter/appsettings.json index fd78b76088..fde5467e3f 100644 --- a/eng/tools/ToolMetadataExporter/appsettings.json +++ b/eng/tools/ToolMetadataExporter/appsettings.json @@ -1,9 +1,4 @@ { - "Logging": { - "LogLevel": { - "Default": "Debug" - } - }, "AppConfig": { "IngestionEndpoint": "https://ingest-your-server.region.kusto.windows.net", "QueryEndpoint": "https://your-server.region.kusto.windows.net", From 85307e69b051702a9010cf515ae84bfa32ca5add Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 11:58:55 -0700 Subject: [PATCH 11/53] Update Json serialization to output Enum names. --- .../ToolMetadataExporter/Models/Kusto/McpToolEventType.cs | 5 +++++ .../Models/ModelsSerializationContext.cs | 1 + 2 files changed, 6 insertions(+) diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs index 3c595bbc1b..6bc1ea3c54 100644 --- a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs @@ -1,11 +1,16 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Text.Json.Serialization; + namespace ToolMetadataExporter.Models.Kusto; public enum McpToolEventType { + [JsonStringEnumMemberName("Created")] Created, + [JsonStringEnumMemberName("Updated")] Updated, + [JsonStringEnumMemberName("Deleted")] Deleted } diff --git a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs index e2b4805f98..4b4facbd1b 100644 --- a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs +++ b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs @@ -9,6 +9,7 @@ namespace ToolMetadataExporter.Models; [JsonSerializable(typeof(McpToolEvent))] [JsonSerializable(typeof(McpToolEventType))] [JsonSerializable(typeof(List))] +[JsonSourceGenerationOptions(Converters = [ typeof(JsonStringEnumConverter )])] public partial class ModelsSerializationContext : JsonSerializerContext { } From dd00c3c13e7e615ff8ff9e0622374b6ee45b1d3f Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 11:59:10 -0700 Subject: [PATCH 12/53] Start host --- eng/tools/ToolMetadataExporter/Program.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 2ff963d969..1f1545a607 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -29,8 +29,9 @@ public static async Task Main(string[] args) var analyzer = host.Services.GetRequiredService(); + await host.StartAsync(); + await analyzer.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); - await host.RunAsync(); } private static void ConfigureServices(IServiceCollection services, IConfiguration configuration) From cab589b4141e02b0a2356d42cc455a92d8593131 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 11:59:42 -0700 Subject: [PATCH 13/53] Move JsonOutput parsing. Fix command line arguments to pass to exe --- eng/tools/ToolMetadataExporter/Utility.cs | 55 ++++++++++++++--------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs index a7c027b0e5..ff67dda914 100644 --- a/eng/tools/ToolMetadataExporter/Utility.cs +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -14,21 +14,9 @@ internal class Utility try { var output = await ExecuteAzmcpAsync("tools list", isCiMode); - // Filter out non-JSON lines (like launch settings messages) - var lines = output.Split('\n'); - var jsonStartIndex = -1; + var jsonOutput = GetJsonFromOutput(output); - for (int i = 0; i < lines.Length; i++) - { - if (lines[i].Trim().StartsWith("{")) - { - jsonStartIndex = i; - - break; - } - } - - if (jsonStartIndex == -1) + if (jsonOutput == null) { if (isCiMode) { @@ -38,8 +26,6 @@ internal class Utility throw new InvalidOperationException("No JSON output found from azmcp command."); } - var jsonOutput = string.Join('\n', lines.Skip(jsonStartIndex)); - // Parse the JSON output var result = JsonSerializer.Deserialize(jsonOutput, SourceGenerationContext.Default.ListToolsResult); @@ -66,10 +52,11 @@ internal class Utility internal static async Task GetVersionAsync() { - return await ExecuteAzmcpAsync("version"); + var output = await ExecuteAzmcpAsync("--version", checkErrorCode: false); + return output.Trim(); } - internal static async Task ExecuteAzmcpAsync(string arguments, bool isCiMode = false) + internal static async Task ExecuteAzmcpAsync(string arguments, bool isCiMode = false, bool checkErrorCode = true) { // Locate azmcp artifact across common build outputs (servers/core, Debug/Release) var exeDir = AppContext.BaseDirectory; @@ -115,7 +102,7 @@ internal static async Task ExecuteAzmcpAsync(string arguments, bool isCi var isDll = string.Equals(cliArtifact.Extension, ".dll", StringComparison.OrdinalIgnoreCase); var fileName = isDll ? "dotnet" : cliArtifact.FullName; - var argumentsToUse = isDll ? $"{cliArtifact.FullName} " : "tools list"; + var argumentsToUse = isDll ? $"{cliArtifact.FullName} " : arguments; var process = new Process { @@ -137,7 +124,7 @@ internal static async Task ExecuteAzmcpAsync(string arguments, bool isCi await process.WaitForExitAsync(); - if (process.ExitCode != 0) + if (checkErrorCode && process.ExitCode != 0) { if (isCiMode) { @@ -177,6 +164,34 @@ internal static async Task ExecuteAzmcpAsync(string arguments, bool isCi return result; } + private static string? GetJsonFromOutput(string? output) + { + if (output == null) + { + return null; + } + + // Filter out non-JSON lines (like launch settings messages) + var lines = output.Split('\n'); + var jsonStartIndex = -1; + + for (int i = 0; i < lines.Length; i++) + { + if (lines[i].Trim().StartsWith("{")) + { + jsonStartIndex = i; + + break; + } + } + + if (jsonStartIndex == -1) + { + return null; + } + + return string.Join('\n', lines.Skip(jsonStartIndex)); + } private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, string filePath) { From 43156a53dbf78f64440f0e52c15a70da9bc1b0f7 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 12:00:05 -0700 Subject: [PATCH 14/53] Pass in command rather than just name in changes. --- eng/tools/ToolMetadataExporter/ToolAnalyzer.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs index 6e1b67a978..57d95a1674 100644 --- a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -9,6 +9,8 @@ namespace ToolMetadataExporter; public class ToolAnalyzer { + private const string Separator = "_"; + private readonly AzmcpProgram _azmcpExe; private readonly IAzureMcpDatastore _azureMcpDatastore; private readonly ILogger _logger; @@ -68,7 +70,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat var toolArea = GetToolArea(tool); if (string.IsNullOrEmpty(toolArea)) { - throw new InvalidOperationException($"Tool without a tool area. Name: {tool.Name}. Id: {tool.Id}"); + throw new InvalidOperationException($"Tool without a tool area. Name: {tool.Name}. Command: {tool.Command} Id: {tool.Id}"); } var changeEvent = new McpToolEvent @@ -78,10 +80,12 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat ServerVersion = serverVersion, }; + var commandWithSeparator = tool.Command?.Replace(" ", Separator); + var hasChange = false; if (existingTools.Remove(tool.Id, out var knownValue)) { - if (!string.Equals(tool.Name, knownValue.ToolName, StringComparison.OrdinalIgnoreCase) + if (!string.Equals(commandWithSeparator, knownValue.ToolName, StringComparison.OrdinalIgnoreCase) || !string.Equals(toolArea, knownValue.ToolArea, StringComparison.OrdinalIgnoreCase)) { hasChange = true; @@ -89,7 +93,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat changeEvent.EventType = McpToolEventType.Updated; changeEvent.ToolName = knownValue.ToolName; changeEvent.ToolArea = knownValue.ToolArea; - changeEvent.ReplacedByToolName = tool.Name; + changeEvent.ReplacedByToolName = commandWithSeparator; changeEvent.ReplacedByToolArea = toolArea; } } @@ -97,7 +101,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat { hasChange = true; changeEvent.EventType = McpToolEventType.Created; - changeEvent.ToolName = tool.Name; + changeEvent.ToolName = commandWithSeparator; changeEvent.ToolArea = toolArea; } From c185cc2f5489882ee5880c9d81665cd1b11a9f28 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 13:54:20 -0700 Subject: [PATCH 15/53] Update Logging --- eng/tools/ToolMetadataExporter/ToolAnalyzer.cs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs index 57d95a1674..c96503d913 100644 --- a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -29,7 +29,7 @@ public ToolAnalyzer(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, CancellationToken cancellationToken = default) { - _logger.LogInformation("Starting analysis."); + _logger.LogInformation("Starting analysis. IsDryRun: {IsDryRun}", isDryRun); var serverVersion = await _azmcpExe.GetVersionAsync(); var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); @@ -67,7 +67,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat throw new InvalidOperationException($"Tool without an id. Name: {tool.Name}. Command: {tool.Command}"); } - var toolArea = GetToolArea(tool); + var toolArea = GetToolArea(tool)?.ToLowerInvariant(); if (string.IsNullOrEmpty(toolArea)) { throw new InvalidOperationException($"Tool without a tool area. Name: {tool.Name}. Command: {tool.Command} Id: {tool.Id}"); @@ -80,7 +80,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat ServerVersion = serverVersion, }; - var commandWithSeparator = tool.Command?.Replace(" ", Separator); + var commandWithSeparator = tool.Command?.Replace(" ", Separator).ToLowerInvariant(); var hasChange = false; if (existingTools.Remove(tool.Id, out var knownValue)) @@ -137,8 +137,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat var outputFile = Path.Combine(_workingDirectory, "tool_changes.json"); - _logger.LogInformation("Tool updates. Writing output to : {FileName}", outputFile); - + _logger.LogInformation("Tool updates. Writing output to: {FileName}", outputFile); var writerOptions = new JsonWriterOptions { @@ -152,7 +151,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat try { - await File.WriteAllBytesAsync(outputFile, ms.ToArray()); + await File.WriteAllBytesAsync(outputFile, ms.ToArray(), cancellationToken); } catch (Exception ex) { @@ -172,5 +171,4 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat var split = tool.Command?.Split(" ", 2); return split == null ? null : split[0]; } - } From b24854a4e54244e116b10306713c5528daec16fb Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 13:57:01 -0700 Subject: [PATCH 16/53] Change interface to use List. Update Ingestion settings for singlejson --- .../Services/AzureMcpKustoDatastore.cs | 23 ++++++++++++++++--- .../Services/IAzureMcpDatastore.cs | 2 +- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs index beb756234a..258e4010a0 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -91,7 +91,7 @@ public async Task> GetAvailableToolsAsync(CancellationToken return results; } - public async Task AddToolEventsAsync(IList toolEvents, CancellationToken cancellationToken = default) + public async Task AddToolEventsAsync(List toolEvents, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -104,7 +104,7 @@ public async Task AddToolEventsAsync(IList toolEvents, Cancellatio var ingestionProperties = new KustoIngestionProperties(_databaseName, _tableName) { - Format = DataSourceFormat.json, + Format = DataSourceFormat.singlejson, IngestionMapping = new IngestionMapping() { IngestionMappingKind = IngestionMappingKind.Json, @@ -112,7 +112,24 @@ public async Task AddToolEventsAsync(IList toolEvents, Cancellatio } }; - await _ingestClient.IngestFromStreamAsync(stream, ingestionProperties); + var result = await _ingestClient.IngestFromStreamAsync(stream, ingestionProperties); + + if (result != null) + { + _logger.LogInformation("Ingestion results."); + foreach(var item in result.GetIngestionStatusCollection()) + { + _logger.LogInformation("- {IngestionSourceId}\t{Table}\t{Status}\t{Details}", + item.IngestionSourceId, + item.Table, + item.Status, + item.Details); + } + } + else + { + _logger.LogWarning("Ingestion client did not produce any results."); + } } internal async IAsyncEnumerable GetLatestToolEventsAsync(string kqlFilePath, diff --git a/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs b/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs index ace467be80..6a20206c51 100644 --- a/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs @@ -10,5 +10,5 @@ public interface IAzureMcpDatastore { Task> GetAvailableToolsAsync(CancellationToken cancellationToken = default); - Task AddToolEventsAsync(IList toolEvents, CancellationToken cancellationToken = default); + Task AddToolEventsAsync(List toolEvents, CancellationToken cancellationToken = default); } From 218c0052ef43aef706a4a9e17f244240c6c5a420 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 13:57:22 -0700 Subject: [PATCH 17/53] Fix column mappings --- .../ToolMetadataExporter/Models/Kusto/McpToolEvent.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs index 0adc12a18e..fbbe3b90b1 100644 --- a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs @@ -44,13 +44,14 @@ public class McpToolEvent public static ColumnMapping[] GetColumnMappings() { return [ - new ColumnMapping { ColumnName = EventTypeColumn, ColumnType = "datetime" }, + new ColumnMapping { ColumnName = EventTimeColumn, ColumnType = "datetime" }, + new ColumnMapping { ColumnName = EventTypeColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ReplacedByToolAreaColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ReplacedByToolNameColumn, ColumnType = "string"}, new ColumnMapping { ColumnName = ServerVersionColumn, ColumnType = "string" }, + new ColumnMapping { ColumnName = ToolAreaColumn , ColumnType = "string" }, new ColumnMapping { ColumnName = ToolIdColumn, ColumnType = "string"}, new ColumnMapping { ColumnName = ToolNameColumn, ColumnType = "string" }, - new ColumnMapping { ColumnName = ToolAreaColumn , ColumnType = "string" }, - new ColumnMapping { ColumnName = ReplacedByToolNameColumn, ColumnType = "string"}, - new ColumnMapping { ColumnName = ReplacedByToolAreaColumn, ColumnType = "string"}, ]; } } From eb6feb42416b6d9c39053ecce689b32514e73628 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 13:58:36 -0700 Subject: [PATCH 18/53] Change to use Direct Ingestion client --- eng/tools/ToolMetadataExporter/Program.cs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 1f1545a607..190adfa557 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -31,7 +31,14 @@ public static async Task Main(string[] args) await host.StartAsync(); - await analyzer.RunAsync(DateTimeOffset.UtcNow, isDryRun: true); + var isDryRunValue = builder.Configuration["IsDryRun"]; + var isDryRun = false; + if (bool.TryParse(isDryRunValue, out var parsed)) + { + isDryRun = parsed; + } + + await analyzer.RunAsync(DateTimeOffset.UtcNow, isDryRun); } private static void ConfigureServices(IServiceCollection services, IConfiguration configuration) @@ -70,19 +77,21 @@ private static void ConfigureAzureServices(IServiceCollection services) services.AddSingleton(sp => { var config = sp.GetRequiredService>(); - //var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.QueryEndpoint) + .WithAadUserPromptAuthentication() .WithAadAzCliAuthentication(interactive: true); return KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder); }); services.AddSingleton(sp => { var config = sp.GetRequiredService>(); - //var credential = sp.GetRequiredService(); + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.IngestionEndpoint) - .WithAadAzCliAuthentication(interactive: true); + .WithAadUserPromptAuthentication() + .WithAadAzCliAuthentication(interactive: true); - return KustoIngestFactory.CreateQueuedIngestClient(connectionStringBuilder); + return KustoIngestFactory.CreateDirectIngestClient(connectionStringBuilder); }); } } From b3683860a50c693c4e7d53b0341ba2d3994a84d2 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 19:59:18 -0700 Subject: [PATCH 19/53] Fix formatting issues --- eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs | 4 ++-- eng/tools/ToolMetadataExporter/Program.cs | 6 ++++-- .../ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs b/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs index b4a68bb118..781922e64e 100644 --- a/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs +++ b/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs @@ -81,8 +81,8 @@ public class ToolAnnotations } // Tool definition for azmcp tools list response -public class Tool { - +public class Tool +{ [JsonPropertyName("id")] public string? Id { get; set; } diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/Program.cs index 190adfa557..01151c9e5a 100644 --- a/eng/tools/ToolMetadataExporter/Program.cs +++ b/eng/tools/ToolMetadataExporter/Program.cs @@ -66,7 +66,8 @@ private static void ConfigureServices(IServiceCollection services, IConfiguratio private static void ConfigureAzureServices(IServiceCollection services) { - services.AddScoped(sp => { + services.AddScoped(sp => + { var credential = new ChainedTokenCredential( new ManagedIdentityCredential(), new DefaultAzureCredential() @@ -84,7 +85,8 @@ private static void ConfigureAzureServices(IServiceCollection services) return KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder); }); - services.AddSingleton(sp => { + services.AddSingleton(sp => + { var config = sp.GetRequiredService>(); var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.IngestionEndpoint) diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs index 258e4010a0..0e19a151f7 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -117,7 +117,7 @@ public async Task AddToolEventsAsync(List toolEvents, Cancellation if (result != null) { _logger.LogInformation("Ingestion results."); - foreach(var item in result.GetIngestionStatusCollection()) + foreach (var item in result.GetIngestionStatusCollection()) { _logger.LogInformation("- {IngestionSourceId}\t{Table}\t{Status}\t{Details}", item.IngestionSourceId, From b1a39ba4b1c6c8b256743d0e82682ab0d3024f7a Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 20:50:13 -0700 Subject: [PATCH 20/53] Fix formatting --- .../ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs index 0e19a151f7..45fbec2f0e 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -119,7 +119,7 @@ public async Task AddToolEventsAsync(List toolEvents, Cancellation _logger.LogInformation("Ingestion results."); foreach (var item in result.GetIngestionStatusCollection()) { - _logger.LogInformation("- {IngestionSourceId}\t{Table}\t{Status}\t{Details}", + _logger.LogInformation("Id: {IngestionSourceId}\tTable: {Table}\tStatus: {Status}\tDetails: {Details}", item.IngestionSourceId, item.Table, item.Status, From 518db947afaa15530b9fa1ee06df4adddd6815ae Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 20:50:23 -0700 Subject: [PATCH 21/53] Add Friend assembly --- eng/tools/ToolMetadataExporter/AssemblyInfo.cs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/AssemblyInfo.cs diff --git a/eng/tools/ToolMetadataExporter/AssemblyInfo.cs b/eng/tools/ToolMetadataExporter/AssemblyInfo.cs new file mode 100644 index 0000000000..e0e0f77159 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/AssemblyInfo.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ToolMetadataExporter.UnitTests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] From addcfcc470dd754e9b65059d1ed9cf61206f8298 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 20:50:33 -0700 Subject: [PATCH 22/53] Add unit test project --- .../ToolMetadataExporter.UnitTests.csproj | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj diff --git a/eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj b/eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj new file mode 100644 index 0000000000..08c835a03d --- /dev/null +++ b/eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj @@ -0,0 +1,16 @@ + + + true + Exe + + + + + + + + + + + + From 773352927f8607b66e209cac463abf2b19c2c188 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 27 Oct 2025 21:00:12 -0700 Subject: [PATCH 23/53] Make method virtual --- eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs index 6a645bb567..c4491ece4c 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -17,7 +17,7 @@ public AzmcpProgram(IOptions options) return Utility.LoadToolsDynamicallyAsync(_toolDirectory, false); } - public Task GetVersionAsync() + public virtual Task GetVersionAsync() { return Utility.GetVersionAsync(); } From 3ba8162ea1e395717f851df887bd160b02142074 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 28 Oct 2025 02:39:27 -0700 Subject: [PATCH 24/53] Fix formatting issue. --- .../ToolMetadataExporter/Models/ModelsSerializationContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs index 4b4facbd1b..e4ccddbb36 100644 --- a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs +++ b/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs @@ -9,7 +9,7 @@ namespace ToolMetadataExporter.Models; [JsonSerializable(typeof(McpToolEvent))] [JsonSerializable(typeof(McpToolEventType))] [JsonSerializable(typeof(List))] -[JsonSourceGenerationOptions(Converters = [ typeof(JsonStringEnumConverter )])] +[JsonSourceGenerationOptions(Converters = [typeof(JsonStringEnumConverter)])] public partial class ModelsSerializationContext : JsonSerializerContext { } From c66ffe8a2cfd6ae1ca2e5a8e94662079f0f1338e Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 28 Oct 2025 02:39:42 -0700 Subject: [PATCH 25/53] Add header --- eng/tools/ToolMetadataExporter/ToolAnalyzer.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs index c96503d913..22c7764347 100644 --- a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ToolMetadataExporter.Models; using ToolMetadataExporter.Models.Kusto; From b8f89090297585799bdc33227089852bd3a32a4e Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 28 Oct 2025 09:41:57 -0700 Subject: [PATCH 26/53] Add dev settings --- .../appsettings.Development.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/appsettings.Development.json diff --git a/eng/tools/ToolMetadataExporter/appsettings.Development.json b/eng/tools/ToolMetadataExporter/appsettings.Development.json new file mode 100644 index 0000000000..0203ba7a47 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/appsettings.Development.json @@ -0,0 +1,12 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Debug" + } + }, + "AppConfig": { + "IngestionEndpoint": "https://ingest-test.westus.kusto.windows.net", + "QueryEndpoint": "https://test.westus.kusto.windows.net", + "DatabaseName": "McpToolTest" + } +} From 7bd7bd9885aa35aab9721afae8583adb97a8b747 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Wed, 29 Oct 2025 01:51:55 -0700 Subject: [PATCH 27/53] Add header --- eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs b/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs index 60cc77c698..26dfecd3f6 100644 --- a/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs +++ b/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs @@ -1,4 +1,7 @@ -namespace ToolMetadataExporter.Models; +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter.Models; public record AzureMcpTool( string ToolId, From 398ad9d144725018e50c25393fea58ea4967c39d Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 11 Nov 2025 11:28:00 -0800 Subject: [PATCH 28/53] Add ServerName support. --- .../ToolMetadataExporter/Models/Kusto/McpToolEvent.cs | 4 ++++ eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs | 5 +++++ eng/tools/ToolMetadataExporter/ToolAnalyzer.cs | 3 +++ eng/tools/ToolMetadataExporter/Utility.cs | 7 ++++++- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs index fbbe3b90b1..c034594429 100644 --- a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs +++ b/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs @@ -10,6 +10,7 @@ public class McpToolEvent { private const string EventTimeColumn = "EventTime"; private const string EventTypeColumn = "EventType"; + private const string ServerNameColumn = "ServerName"; private const string ServerVersionColumn = "ServerVersion"; private const string ToolIdColumn = "ToolId"; private const string ToolNameColumn = "ToolName"; @@ -23,6 +24,9 @@ public class McpToolEvent [JsonPropertyName(EventTypeColumn)] public McpToolEventType? EventType { get; set; } + [JsonPropertyName(ServerNameColumn)] + public string? ServerName { get; set; } + [JsonPropertyName(ServerVersionColumn)] public string? ServerVersion { get; set; } diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs index c4491ece4c..8a0d433766 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -7,6 +7,11 @@ public class AzmcpProgram { private readonly string _toolDirectory; + public Task GetServerNameAsync() + { + return Task.FromResult(""); + } + public AzmcpProgram(IOptions options) { _toolDirectory = options.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs index 22c7764347..101fbd5f08 100644 --- a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs @@ -34,6 +34,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat { _logger.LogInformation("Starting analysis. IsDryRun: {IsDryRun}", isDryRun); + var serverName = await _azmcpExe.GetServerNameAsync(); var serverVersion = await _azmcpExe.GetVersionAsync(); var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); @@ -80,6 +81,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat { EventTime = analysisTime, ToolId = tool.Id, + ServerName = serverName, ServerVersion = serverVersion, }; @@ -118,6 +120,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat // Any remaining entries in `existingTool` are ones that got deleted. var removals = existingTools.Select(x => new McpToolEvent { + ServerName = serverName, ServerVersion = serverVersion, EventTime = analysisTime, EventType = McpToolEventType.Deleted, diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs index ff67dda914..02021da318 100644 --- a/eng/tools/ToolMetadataExporter/Utility.cs +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System.Diagnostics; -using System.Text.Json; using ToolSelection.Models; namespace ToolMetadataExporter; @@ -50,6 +49,12 @@ internal class Utility } } + internal static async Task GetServerName() + { + var output = await ExecuteAzmcpAsync("", checkErrorCode: false); + return output.Trim(); + } + internal static async Task GetVersionAsync() { var output = await ExecuteAzmcpAsync("--version", checkErrorCode: false); From fcadc9efabbc14c2ee6dd35a86ad1b2ca85c2454 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 11 Nov 2025 11:28:22 -0800 Subject: [PATCH 29/53] Remove test template. --- .../ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs diff --git a/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs b/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs deleted file mode 100644 index 4882f1233f..0000000000 --- a/eng/tools/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace ToolMetadataExporter.UnitTests; - -public class ToolAnalyzerTests -{ -} From fc3e09bdaadcf4ee5ac3125a234de9cd94c279a8 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 11 Nov 2025 11:32:47 -0800 Subject: [PATCH 30/53] Add ServerName --- .../Services/AzmcpProgram.cs | 2 +- eng/tools/ToolMetadataExporter/Utility.cs | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs index 8a0d433766..19585d61e9 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs @@ -9,7 +9,7 @@ public class AzmcpProgram public Task GetServerNameAsync() { - return Task.FromResult(""); + return Utility.GetServerName(); } public AzmcpProgram(IOptions options) diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/Utility.cs index 02021da318..f09c63c6d1 100644 --- a/eng/tools/ToolMetadataExporter/Utility.cs +++ b/eng/tools/ToolMetadataExporter/Utility.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System.Diagnostics; +using System.Text.RegularExpressions; using ToolSelection.Models; namespace ToolMetadataExporter; @@ -51,8 +52,19 @@ internal class Utility internal static async Task GetServerName() { - var output = await ExecuteAzmcpAsync("", checkErrorCode: false); - return output.Trim(); + var output = await ExecuteAzmcpAsync("--help", checkErrorCode: false); + + string[] array = Regex.Split(output, "\n\r"); + for (int i = 0; i < array.Length; i++) + { + string? line = array[i]; + if (line.StartsWith("Description:")) + { + return array[i + 1].Trim(); + } + } + + throw new InvalidOperationException("Could not find server name"); } internal static async Task GetVersionAsync() From 1e7eee54296a90166fc2fbc85d86c1e9586e1491 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 11 Nov 2025 11:35:46 -0800 Subject: [PATCH 31/53] Propagate cancellation --- .../ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs index 45fbec2f0e..5cb7cf42a5 100644 --- a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs +++ b/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs @@ -140,7 +140,7 @@ internal async IAsyncEnumerable GetLatestToolEventsAsync(string kq throw new FileNotFoundException($"KQL file not found: {kqlFilePath}"); } - var kql = await File.ReadAllTextAsync(kqlFilePath); + var kql = await File.ReadAllTextAsync(kqlFilePath, cancellationToken); var clientRequestProperties = new ClientRequestProperties(); var reader = await _kustoClient.ExecuteQueryAsync(_databaseName, kql, clientRequestProperties, cancellationToken); From a527261a6b1dd0f1bd107fa846360d2c2aef6c6a Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Wed, 3 Dec 2025 16:31:00 -0800 Subject: [PATCH 32/53] Moving classes into src/tests --- .../{ => src}/AppConfiguration.cs | 0 .../{ => src}/AssemblyInfo.cs | 0 .../{ => src}/GlobalUsings.cs | 0 .../{ => src}/Models/AzureMcpTool.cs | 0 .../{ => src}/Models/Kusto/McpToolEvent.cs | 0 .../Models/Kusto/McpToolEventType.cs | 0 .../Models/ModelsSerializationContext.cs | 0 .../ToolMetadataExporter/{ => src}/Program.cs | 0 .../{ => src}/Properties/launchSettings.json | 0 .../Resources/queries/CreateTable.kql | 0 .../Resources/queries/GetAvailableTools.kql | 0 .../{ => src}/Services/AzmcpProgram.cs | 0 .../Services/AzureMcpKustoDatastore.cs | 0 .../{ => src}/Services/IAzureMcpDatastore.cs | 0 .../{ => src}/ToolAnalyzer.cs | 0 .../{ => src}/ToolMetadataExporter.csproj | 2 +- .../ToolMetadataExporter/{ => src}/Utility.cs | 0 .../{ => src}/appsettings.Development.json | 0 .../{ => src}/appsettings.json | 0 .../ToolMetadataExporter.UnitTests.csproj | 4 +-- eng/tools/Tools.sln | 29 +++++++++++++++++-- 21 files changed, 30 insertions(+), 5 deletions(-) rename eng/tools/ToolMetadataExporter/{ => src}/AppConfiguration.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/AssemblyInfo.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/GlobalUsings.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Models/AzureMcpTool.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Models/Kusto/McpToolEvent.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Models/Kusto/McpToolEventType.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Models/ModelsSerializationContext.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Program.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Properties/launchSettings.json (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Resources/queries/CreateTable.kql (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Resources/queries/GetAvailableTools.kql (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Services/AzmcpProgram.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Services/AzureMcpKustoDatastore.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/Services/IAzureMcpDatastore.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/ToolAnalyzer.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/ToolMetadataExporter.csproj (91%) rename eng/tools/ToolMetadataExporter/{ => src}/Utility.cs (100%) rename eng/tools/ToolMetadataExporter/{ => src}/appsettings.Development.json (100%) rename eng/tools/ToolMetadataExporter/{ => src}/appsettings.json (100%) rename eng/tools/{ => ToolMetadataExporter/tests}/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj (78%) diff --git a/eng/tools/ToolMetadataExporter/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/src/AppConfiguration.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/AppConfiguration.cs rename to eng/tools/ToolMetadataExporter/src/AppConfiguration.cs diff --git a/eng/tools/ToolMetadataExporter/AssemblyInfo.cs b/eng/tools/ToolMetadataExporter/src/AssemblyInfo.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/AssemblyInfo.cs rename to eng/tools/ToolMetadataExporter/src/AssemblyInfo.cs diff --git a/eng/tools/ToolMetadataExporter/GlobalUsings.cs b/eng/tools/ToolMetadataExporter/src/GlobalUsings.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/GlobalUsings.cs rename to eng/tools/ToolMetadataExporter/src/GlobalUsings.cs diff --git a/eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs b/eng/tools/ToolMetadataExporter/src/Models/AzureMcpTool.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/Models/AzureMcpTool.cs rename to eng/tools/ToolMetadataExporter/src/Models/AzureMcpTool.cs diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs b/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEvent.cs rename to eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs diff --git a/eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs b/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEventType.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/Models/Kusto/McpToolEventType.cs rename to eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEventType.cs diff --git a/eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs b/eng/tools/ToolMetadataExporter/src/Models/ModelsSerializationContext.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/Models/ModelsSerializationContext.cs rename to eng/tools/ToolMetadataExporter/src/Models/ModelsSerializationContext.cs diff --git a/eng/tools/ToolMetadataExporter/Program.cs b/eng/tools/ToolMetadataExporter/src/Program.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/Program.cs rename to eng/tools/ToolMetadataExporter/src/Program.cs diff --git a/eng/tools/ToolMetadataExporter/Properties/launchSettings.json b/eng/tools/ToolMetadataExporter/src/Properties/launchSettings.json similarity index 100% rename from eng/tools/ToolMetadataExporter/Properties/launchSettings.json rename to eng/tools/ToolMetadataExporter/src/Properties/launchSettings.json diff --git a/eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql b/eng/tools/ToolMetadataExporter/src/Resources/queries/CreateTable.kql similarity index 100% rename from eng/tools/ToolMetadataExporter/Resources/queries/CreateTable.kql rename to eng/tools/ToolMetadataExporter/src/Resources/queries/CreateTable.kql diff --git a/eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql b/eng/tools/ToolMetadataExporter/src/Resources/queries/GetAvailableTools.kql similarity index 100% rename from eng/tools/ToolMetadataExporter/Resources/queries/GetAvailableTools.kql rename to eng/tools/ToolMetadataExporter/src/Resources/queries/GetAvailableTools.kql diff --git a/eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/Services/AzmcpProgram.cs rename to eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs diff --git a/eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/src/Services/AzureMcpKustoDatastore.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/Services/AzureMcpKustoDatastore.cs rename to eng/tools/ToolMetadataExporter/src/Services/AzureMcpKustoDatastore.cs diff --git a/eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs b/eng/tools/ToolMetadataExporter/src/Services/IAzureMcpDatastore.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/Services/IAzureMcpDatastore.cs rename to eng/tools/ToolMetadataExporter/src/Services/IAzureMcpDatastore.cs diff --git a/eng/tools/ToolMetadataExporter/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/ToolAnalyzer.cs rename to eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs diff --git a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj b/eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj similarity index 91% rename from eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj rename to eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj index 39d1ece9b1..b3e4cb42b4 100644 --- a/eng/tools/ToolMetadataExporter/ToolMetadataExporter.csproj +++ b/eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj @@ -9,7 +9,7 @@ - + diff --git a/eng/tools/ToolMetadataExporter/Utility.cs b/eng/tools/ToolMetadataExporter/src/Utility.cs similarity index 100% rename from eng/tools/ToolMetadataExporter/Utility.cs rename to eng/tools/ToolMetadataExporter/src/Utility.cs diff --git a/eng/tools/ToolMetadataExporter/appsettings.Development.json b/eng/tools/ToolMetadataExporter/src/appsettings.Development.json similarity index 100% rename from eng/tools/ToolMetadataExporter/appsettings.Development.json rename to eng/tools/ToolMetadataExporter/src/appsettings.Development.json diff --git a/eng/tools/ToolMetadataExporter/appsettings.json b/eng/tools/ToolMetadataExporter/src/appsettings.json similarity index 100% rename from eng/tools/ToolMetadataExporter/appsettings.json rename to eng/tools/ToolMetadataExporter/src/appsettings.json diff --git a/eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj similarity index 78% rename from eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj rename to eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj index 08c835a03d..49f3f59f9c 100644 --- a/eng/tools/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj @@ -1,4 +1,4 @@ - + true Exe @@ -11,6 +11,6 @@ - + diff --git a/eng/tools/Tools.sln b/eng/tools/Tools.sln index febcf965b4..5267747a7a 100644 --- a/eng/tools/Tools.sln +++ b/eng/tools/Tools.sln @@ -6,6 +6,18 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ToolDescriptionEvaluator", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolDescriptionEvaluator", "ToolDescriptionEvaluator\src\ToolDescriptionEvaluator.csproj", "{CE346AEC-0DD6-C210-A058-B6D879AF55B1}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ToolMetadataExporter", "ToolMetadataExporter", "{F86DD719-7EB8-4D49-81EA-F2D6565A7047}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{B8D43586-6D54-4A71-A714-D7AC92AE64DD}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{B28D2E97-9CDA-4BF2-95EC-CF169C6339EC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter", "ToolMetadataExporter\src\ToolMetadataExporter.csproj", "{41A27DC4-E2C0-E36D-EF8E-854DFC376331}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter.UnitTests", "ToolMetadataExporter\tests\ToolMetadataExporter.UnitTests\ToolMetadataExporter.UnitTests.csproj", "{8B9656E4-84BE-0199-1E72-B4E5B713E059}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -16,14 +28,27 @@ Global {CE346AEC-0DD6-C210-A058-B6D879AF55B1}.Debug|Any CPU.Build.0 = Debug|Any CPU {CE346AEC-0DD6-C210-A058-B6D879AF55B1}.Release|Any CPU.ActiveCfg = Release|Any CPU {CE346AEC-0DD6-C210-A058-B6D879AF55B1}.Release|Any CPU.Build.0 = Release|Any CPU + {41A27DC4-E2C0-E36D-EF8E-854DFC376331}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {41A27DC4-E2C0-E36D-EF8E-854DFC376331}.Debug|Any CPU.Build.0 = Debug|Any CPU + {41A27DC4-E2C0-E36D-EF8E-854DFC376331}.Release|Any CPU.ActiveCfg = Release|Any CPU + {41A27DC4-E2C0-E36D-EF8E-854DFC376331}.Release|Any CPU.Build.0 = Release|Any CPU + {8B9656E4-84BE-0199-1E72-B4E5B713E059}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8B9656E4-84BE-0199-1E72-B4E5B713E059}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8B9656E4-84BE-0199-1E72-B4E5B713E059}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8B9656E4-84BE-0199-1E72-B4E5B713E059}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution - {CE346AEC-0DD6-C210-A058-B6D879AF55B1} = {89C7D52E-46CB-E5E4-EB56-C23691DF38B5} + {CE346AEC-0DD6-C210-A058-B6D879AF55B1} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} + {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} = {89C7D52E-46CB-E5E4-EB56-C23691DF38B5} + {B8D43586-6D54-4A71-A714-D7AC92AE64DD} = {F86DD719-7EB8-4D49-81EA-F2D6565A7047} + {B28D2E97-9CDA-4BF2-95EC-CF169C6339EC} = {F86DD719-7EB8-4D49-81EA-F2D6565A7047} + {41A27DC4-E2C0-E36D-EF8E-854DFC376331} = {B8D43586-6D54-4A71-A714-D7AC92AE64DD} + {8B9656E4-84BE-0199-1E72-B4E5B713E059} = {B28D2E97-9CDA-4BF2-95EC-CF169C6339EC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FE4B2877-CB8E-40CE-92EA-A30A5B9C07EE} EndGlobalSection -EndGlobal \ No newline at end of file +EndGlobal From 5ab55abaadc1aeb0ab61f270003fdae450951f32 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Tue, 6 Jan 2026 16:50:57 -0800 Subject: [PATCH 33/53] Move parsing from command line to use IConfiguration. --- .../src/Models/CommandLineOptions.cs | 11 +++++++++++ eng/tools/ToolMetadataExporter/src/Program.cs | 15 +++++++-------- .../src/ToolMetadataExporter.csproj | 2 ++ 3 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs diff --git a/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs b/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs new file mode 100644 index 0000000000..063ab45eae --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter.Models; + +internal class CommandLineOptions +{ + public bool IsDryRun { get; set; } = false; + + public string? AzmcpExe { get; set; } +} diff --git a/eng/tools/ToolMetadataExporter/src/Program.cs b/eng/tools/ToolMetadataExporter/src/Program.cs index 01151c9e5a..69b0fdca9b 100644 --- a/eng/tools/ToolMetadataExporter/src/Program.cs +++ b/eng/tools/ToolMetadataExporter/src/Program.cs @@ -12,6 +12,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; using ToolMetadataExporter.Services; namespace ToolMetadataExporter; @@ -27,18 +28,16 @@ public static async Task Main(string[] args) var host = builder.Build(); - var analyzer = host.Services.GetRequiredService(); - - await host.StartAsync(); + var commandLineOptions = builder.Configuration.Get(); - var isDryRunValue = builder.Configuration["IsDryRun"]; - var isDryRun = false; - if (bool.TryParse(isDryRunValue, out var parsed)) + if (commandLineOptions == null) { - isDryRun = parsed; + throw new InvalidOperationException("Expected to be able to get command line options from IConfiguration."); } - await analyzer.RunAsync(DateTimeOffset.UtcNow, isDryRun); + var analyzer = host.Services.GetRequiredService(); + + await analyzer.RunAsync(DateTimeOffset.UtcNow, commandLineOptions.IsDryRun); } private static void ConfigureServices(IServiceCollection services, IConfiguration configuration) diff --git a/eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj b/eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj index b3e4cb42b4..1b302ecd87 100644 --- a/eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj +++ b/eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj @@ -6,6 +6,8 @@ enable enable true + + true From dd8d9b9b8d04660c80eff86aafb40be1bd029e49 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 8 Jan 2026 14:49:35 -0800 Subject: [PATCH 34/53] Add ServerName column to Kusto mapping at the end. --- eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs b/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs index c034594429..2c2f27dbc2 100644 --- a/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs +++ b/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs @@ -56,6 +56,7 @@ public static ColumnMapping[] GetColumnMappings() new ColumnMapping { ColumnName = ToolAreaColumn , ColumnType = "string" }, new ColumnMapping { ColumnName = ToolIdColumn, ColumnType = "string"}, new ColumnMapping { ColumnName = ToolNameColumn, ColumnType = "string" }, + new ColumnMapping { ColumnName = ServerNameColumn, ColumnType = "string" }, ]; } } From f868255fd441d022303e892bd92210ece23c1dea Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 8 Jan 2026 14:49:53 -0800 Subject: [PATCH 35/53] Add documentation to CommandLineOptions --- .../src/Models/CommandLineOptions.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs b/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs index 063ab45eae..a5123f8cb6 100644 --- a/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs +++ b/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs @@ -3,9 +3,24 @@ namespace ToolMetadataExporter.Models; +/// +/// Options specified via command line arguments. Supported options are: +/// +/// --dry-run: If specified, the tool will run in dry-run mode, meaning no changes will be made to the target datastore. +/// --azmcp-exe <path>: The path to the azmcp executable to use for interacting with the MCP server. +/// +/// internal class CommandLineOptions { - public bool IsDryRun { get; set; } = false; + /// + /// Gets or sets a value indicating whether the tool analysis should be performed as a dry run. + /// + /// When set to , the operation is performed, output to the console, but not persisted to the datastore. When set to + /// , the operation is executed normally. + public bool? IsDryRun { get; set; } + /// + /// Gets or sets the full path to the AzMcp executable file. + /// public string? AzmcpExe { get; set; } } From 9b88050d34759674038748878f55a992b7989950 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 8 Jan 2026 14:52:13 -0800 Subject: [PATCH 36/53] Add ServerInfoResult for `server info` --- .../src/Models/ModelsSerializationContext.cs | 2 ++ .../src/Models/ServerInfoResult.cs | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/src/Models/ServerInfoResult.cs diff --git a/eng/tools/ToolMetadataExporter/src/Models/ModelsSerializationContext.cs b/eng/tools/ToolMetadataExporter/src/Models/ModelsSerializationContext.cs index e4ccddbb36..ac06218715 100644 --- a/eng/tools/ToolMetadataExporter/src/Models/ModelsSerializationContext.cs +++ b/eng/tools/ToolMetadataExporter/src/Models/ModelsSerializationContext.cs @@ -6,6 +6,8 @@ namespace ToolMetadataExporter.Models; +[JsonSerializable(typeof(ServerInfo))] +[JsonSerializable(typeof(ServerInfoResult))] [JsonSerializable(typeof(McpToolEvent))] [JsonSerializable(typeof(McpToolEventType))] [JsonSerializable(typeof(List))] diff --git a/eng/tools/ToolMetadataExporter/src/Models/ServerInfoResult.cs b/eng/tools/ToolMetadataExporter/src/Models/ServerInfoResult.cs new file mode 100644 index 0000000000..ba90e568b5 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Models/ServerInfoResult.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace ToolMetadataExporter.Models; + +/// +/// The result of a `server info` request from the MCP server. +/// +public class ServerInfoResult +{ + [JsonPropertyName("status")] + + public int Status { get; set; } + + [JsonPropertyName("message")] + public string? Message { get; set; } + + [JsonPropertyName("results")] + public ServerInfo? Results { get; set; } +} + +public class ServerInfo +{ + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("version")] + public string Version { get; set; } = string.Empty; +} From dd909a6f13885e12f4f5c06c798c45329a7530cd Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 8 Jan 2026 14:52:53 -0800 Subject: [PATCH 37/53] Add IsDryRun, AzMcp to app configuration --- eng/tools/ToolMetadataExporter/src/AppConfiguration.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eng/tools/ToolMetadataExporter/src/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/src/AppConfiguration.cs index 915802ae62..921fe6de18 100644 --- a/eng/tools/ToolMetadataExporter/src/AppConfiguration.cs +++ b/eng/tools/ToolMetadataExporter/src/AppConfiguration.cs @@ -16,4 +16,10 @@ public class AppConfiguration public string? QueriesFolder { get; set; } = "Resources/queries"; public string? WorkDirectory { get; set; } + + public bool IsDryRun { get; set; } + + public string? AzmcpExe { get; set; } + + public bool IsAzmcpExeSpecified { get; set; } } From 1b1449afb2222546cfa135d90dfee016e415b9ee Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 8 Jan 2026 14:53:16 -0800 Subject: [PATCH 38/53] Update Program to configure AppConfiguration and CommandLineOptions --- eng/tools/ToolMetadataExporter/src/Program.cs | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/src/Program.cs b/eng/tools/ToolMetadataExporter/src/Program.cs index 69b0fdca9b..765bab3d1d 100644 --- a/eng/tools/ToolMetadataExporter/src/Program.cs +++ b/eng/tools/ToolMetadataExporter/src/Program.cs @@ -27,17 +27,9 @@ public static async Task Main(string[] args) ConfigureAzureServices(builder.Services); var host = builder.Build(); - - var commandLineOptions = builder.Configuration.Get(); - - if (commandLineOptions == null) - { - throw new InvalidOperationException("Expected to be able to get command line options from IConfiguration."); - } - var analyzer = host.Services.GetRequiredService(); - await analyzer.RunAsync(DateTimeOffset.UtcNow, commandLineOptions.IsDryRun); + await analyzer.RunAsync(DateTimeOffset.UtcNow); } private static void ConfigureServices(IServiceCollection services, IConfiguration configuration) @@ -51,14 +43,42 @@ private static void ConfigureServices(IServiceCollection services, IConfiguratio .AddSingleton() .AddSingleton(); - services.Configure(configuration.GetSection("AppConfig")) - .PostConfigure(existing => + services.AddOptions() + .Bind(configuration); + + services.AddOptions() + .Bind(configuration) + .Configure>((existing, commandLineOptions) => { - if (existing.WorkDirectory == null) + // Command-line IsDryRun overrides appsettings.json file value. + if (commandLineOptions.Value.IsDryRun.HasValue) + { + existing.IsDryRun = commandLineOptions.Value.IsDryRun.Value; + } + + var exeDir = AppContext.BaseDirectory; + + // If a path to azmcp.exe is not provided. Assume that this is running within the context of + // the repository and try to find it. + existing.IsAzmcpExeSpecified = !string.IsNullOrEmpty(commandLineOptions.Value.AzmcpExe); + if (existing.IsAzmcpExeSpecified) + { + existing.AzmcpExe = commandLineOptions.Value.AzmcpExe!; + + if (existing.WorkDirectory == null) + { + existing.WorkDirectory = exeDir; + } + } + else { - string exeDir = AppContext.BaseDirectory; var repoRoot = Utility.FindRepoRoot(exeDir); - existing.WorkDirectory = Path.Combine(repoRoot, ".work"); + if (existing.WorkDirectory == null) + { + existing.WorkDirectory = Path.Combine(repoRoot, ".work"); + } + + existing.AzmcpExe = Path.Combine(repoRoot, "eng", "tools", "Azmcp", "azmcp.exe"); } }); } From dd6dad33c56427324b0aa6ed3eae09ba85717c47 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 8 Jan 2026 14:53:55 -0800 Subject: [PATCH 39/53] Add FileName parameter to utility methods. --- .../src/Services/AzmcpProgram.cs | 45 ++++++++++++++---- eng/tools/ToolMetadataExporter/src/Utility.cs | 46 ++++++++++--------- 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs index 19585d61e9..c0b2a8cfc2 100644 --- a/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs @@ -1,29 +1,56 @@ -using Microsoft.Extensions.Options; +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; using ToolSelection.Models; namespace ToolMetadataExporter.Services; +/// +/// Represents the MCP server and exposes methods to interact with it. +/// public class AzmcpProgram { private readonly string _toolDirectory; + private readonly string _azureMcp; + private readonly ILogger _logger; + private readonly Lazy> - public Task GetServerNameAsync() + public AzmcpProgram(IOptions options, ILogger logger) { - return Utility.GetServerName(); + _toolDirectory = options.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); + + _azureMcp = options.Value.AzmcpExe + ?? throw new ArgumentNullException(nameof(CommandLineOptions.AzmcpExe)); + _logger = logger; } - public AzmcpProgram(IOptions options) + /// + /// Gets the name of the MCP server. + /// + /// + public virtual Task GetServerNameAsync() { - _toolDirectory = options.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); + return Utility.GetServerName(_azureMcp); } - public virtual Task LoadToolsDynamicallyAsync() + /// + /// Gets the server version. + /// + /// + public virtual Task GetServerVersionAsync() { - return Utility.LoadToolsDynamicallyAsync(_toolDirectory, false); + return Utility.GetVersionAsync(_azureMcp); } - public virtual Task GetVersionAsync() + /// + /// Gets the list of tools from the MCP server. + /// + /// + public virtual Task LoadToolsDynamicallyAsync() { - return Utility.GetVersionAsync(); + return Utility.LoadToolsDynamicallyAsync(_azureMcp, _toolDirectory, false); } } diff --git a/eng/tools/ToolMetadataExporter/src/Utility.cs b/eng/tools/ToolMetadataExporter/src/Utility.cs index f09c63c6d1..4cf7095ef8 100644 --- a/eng/tools/ToolMetadataExporter/src/Utility.cs +++ b/eng/tools/ToolMetadataExporter/src/Utility.cs @@ -9,11 +9,11 @@ namespace ToolMetadataExporter; internal class Utility { - internal static async Task LoadToolsDynamicallyAsync(string workDirectory, bool isCiMode = false) + internal static async Task LoadToolsDynamicallyAsync(string serverFile, string workDirectory, bool isCiMode = false) { try { - var output = await ExecuteAzmcpAsync("tools list", isCiMode); + var output = await ExecuteAzmcpAsync(serverFile, "tools list", isCiMode); var jsonOutput = GetJsonFromOutput(output); if (jsonOutput == null) @@ -50,9 +50,9 @@ internal class Utility } } - internal static async Task GetServerName() + internal static async Task GetServerName(string serverFile) { - var output = await ExecuteAzmcpAsync("--help", checkErrorCode: false); + var output = await ExecuteAzmcpAsync(serverFile, "--help", checkErrorCode: false); string[] array = Regex.Split(output, "\n\r"); for (int i = 0; i < array.Length; i++) @@ -67,21 +67,18 @@ internal static async Task GetServerName() throw new InvalidOperationException("Could not find server name"); } - internal static async Task GetVersionAsync() + internal static async Task GetVersionAsync(string serverFile) { - var output = await ExecuteAzmcpAsync("--version", checkErrorCode: false); + var output = await ExecuteAzmcpAsync(serverFile, "--version", checkErrorCode: false); return output.Trim(); } - internal static async Task ExecuteAzmcpAsync(string arguments, bool isCiMode = false, bool checkErrorCode = true) + internal static string FindAzmcpAsync(string repositoryRoot, bool isCiMode = false) { - // Locate azmcp artifact across common build outputs (servers/core, Debug/Release) - var exeDir = AppContext.BaseDirectory; - var repoRoot = FindRepoRoot(exeDir); var searchRoots = new List { - Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug"), - Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Release") + Path.Combine(repositoryRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug"), + Path.Combine(repositoryRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Release") }; var candidateNames = new[] { "azmcp.exe", "azmcp", "azmcp.dll" }; @@ -107,19 +104,26 @@ internal static async Task ExecuteAzmcpAsync(string arguments, bool isCi } } - if (cliArtifact == null) + if (cliArtifact != null) { - if (isCiMode) - { - return string.Empty; // Graceful fallback in CI - } + return cliArtifact.FullName; + } - throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); + if (isCiMode) + { + return string.Empty; // Graceful fallback in CI } - var isDll = string.Equals(cliArtifact.Extension, ".dll", StringComparison.OrdinalIgnoreCase); - var fileName = isDll ? "dotnet" : cliArtifact.FullName; - var argumentsToUse = isDll ? $"{cliArtifact.FullName} " : arguments; + throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); + } + + internal static async Task ExecuteAzmcpAsync(string serverFile, string arguments, + bool isCiMode = false, bool checkErrorCode = true) + { + var fileInfo = new FileInfo(serverFile); + var isDll = string.Equals(fileInfo.Extension, ".dll", StringComparison.OrdinalIgnoreCase); + var fileName = isDll ? "dotnet" : fileInfo.FullName; + var argumentsToUse = isDll ? $"{fileInfo.FullName} " : arguments; var process = new Process { From 836c3b3b54aed869dd01477c52b7b42b8f05a77a Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Thu, 8 Jan 2026 14:54:25 -0800 Subject: [PATCH 40/53] Update ToolAnalyzer to use file name parameter from Utility. --- eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs index 101fbd5f08..231a4006ee 100644 --- a/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs @@ -18,6 +18,7 @@ public class ToolAnalyzer private readonly IAzureMcpDatastore _azureMcpDatastore; private readonly ILogger _logger; private readonly string _workingDirectory; + private readonly bool _isDryRun; public ToolAnalyzer(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, IOptions configuration, ILogger logger) @@ -28,14 +29,15 @@ public ToolAnalyzer(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, _workingDirectory = configuration.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); ; + _isDryRun = configuration.Value.IsDryRun; } - public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, CancellationToken cancellationToken = default) + public async Task RunAsync(DateTimeOffset analysisTime, CancellationToken cancellationToken = default) { - _logger.LogInformation("Starting analysis. IsDryRun: {IsDryRun}", isDryRun); + _logger.LogInformation("Starting analysis. IsDryRun: {IsDryRun}", _isDryRun); var serverName = await _azmcpExe.GetServerNameAsync(); - var serverVersion = await _azmcpExe.GetVersionAsync(); + var serverVersion = await _azmcpExe.GetServerVersionAsync(); var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); if (currentTools == null) @@ -165,7 +167,7 @@ public async Task RunAsync(DateTimeOffset analysisTime, bool isDryRun, Cancellat } } - if (!isDryRun) + if (!_isDryRun) { _logger.LogInformation("Updating datastore."); await _azureMcpDatastore.AddToolEventsAsync(changes, cancellationToken); From 01082f595087ced0bc8ad6714522307e44439962 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 12:31:26 -0800 Subject: [PATCH 41/53] Create functions for AzMcp program. --- .../src/Services/AzmcpProgram.cs | 161 ++++++++++++++++-- 1 file changed, 148 insertions(+), 13 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs index c0b2a8cfc2..3eee269556 100644 --- a/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ToolMetadataExporter.Models; @@ -15,42 +16,176 @@ public class AzmcpProgram { private readonly string _toolDirectory; private readonly string _azureMcp; + private readonly Utility _utility; private readonly ILogger _logger; - private readonly Lazy> + private readonly Task _serverInfoTask; + private readonly Task _serverNameTask; + private readonly Task _listToolsTask; + private readonly Task _serverVersionTask; - public AzmcpProgram(IOptions options, ILogger logger) + public AzmcpProgram(IOptions options, Utility utility, ILogger logger) { _toolDirectory = options.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); _azureMcp = options.Value.AzmcpExe ?? throw new ArgumentNullException(nameof(CommandLineOptions.AzmcpExe)); + _utility = utility; _logger = logger; + + _serverInfoTask = GetServerInfoInternalAsync(); + _serverNameTask = GetServerNameInternalAsync(); + _serverVersionTask = GetServerVersionInternalAsync(); + _listToolsTask = GetServerToolsInternalAsync(); } /// - /// Gets the name of the MCP server. + /// Gets the name of the MCP server in lower case. /// /// - public virtual Task GetServerNameAsync() - { - return Utility.GetServerName(_azureMcp); - } + public virtual Task GetServerNameAsync() => _serverNameTask; /// /// Gets the server version. /// /// - public virtual Task GetServerVersionAsync() - { - return Utility.GetVersionAsync(_azureMcp); - } + public virtual Task GetServerVersionAsync() => _serverVersionTask; /// /// Gets the list of tools from the MCP server. /// /// - public virtual Task LoadToolsDynamicallyAsync() + public virtual Task LoadToolsDynamicallyAsync() => _listToolsTask; + + /// + /// Gets the server name of the MCP server in lower-case + /// + /// The server name of the MCP server in lowercase. + /// If name could not be determined. + private async Task GetServerNameInternalAsync() + { + var output = await _serverInfoTask; + + if (output != null) + { + return output.Name.ToLowerInvariant(); + } + + var serverName = await GetServerNameWithHelpCommandAsync(); + + if (serverName == null) + { + throw new InvalidOperationException("Failed to determine server name via `server info` and `--help`."); + } + else + { + return serverName.ToLowerInvariant(); + } + } + + /// + /// Gets the server version of the MCP server. + /// + /// Semver version of the MCP server. + /// If a version could not be determined. + private async Task GetServerVersionInternalAsync() { - return Utility.LoadToolsDynamicallyAsync(_azureMcp, _toolDirectory, false); + var output = await _serverInfoTask; + + if (output != null) + { + return output.Version; + } + else + { + return await InvokeServerVersionCommandAsync(); + } + } + + private Task GetServerToolsInternalAsync() + { + return _utility.LoadToolsDynamicallyAsync(_azureMcp, _toolDirectory, false); + } + + /// + /// Gets server information by invoking the "server info" command in the MCP server. + /// + /// The server information. Null, if output from `server info` could not be parsed from JSON. + /// This may be the case when "server info" has not been implemented by the server. + private async Task GetServerInfoInternalAsync() + { + var output = await _utility.ExecuteAzmcpAsync(_azureMcp, "server info", checkErrorCode: false); + + var result = JsonSerializer.Deserialize(output, ModelsSerializationContext.Default.ServerInfoResult); + + if (result == null || result.Results == null) + { + _logger.LogInformation("The MCP server did not return valid JSON output for the 'server info' command. Output: {Output}", output); + } + + return result?.Results; + } + + /// + /// Invokes the MCP server with the --version argument to get the server version. + /// + /// A semver compatible version. + private async Task InvokeServerVersionCommandAsync() + { + // Invoking --version returns an error code of 1. + var versionOutput = await _utility.ExecuteAzmcpAsync(_azureMcp, "--version", checkErrorCode: false); + + // The version output may contain a git hash after a '+' character. + // Example: "1.0.0+4c6c98bca777f54350e426c01177a2b91ad12fd4" + int hashSeparator = versionOutput.Trim().IndexOf('+'); + if (hashSeparator != -1) + { + versionOutput = versionOutput.Substring(0, hashSeparator); + } + + return versionOutput; + } + + /// + /// Invokes the MCP server with the --help argument to get the server name. + /// + /// The server name for the MCP server. Null if the server name could not be found. + /// When --help is invoked the name is found after the "Description:" line. Spaces in the server + /// name are replaced with periods. Example: "Azure Mcp Server" becomes "Azure.Mcp.Server". + private async Task GetServerNameWithHelpCommandAsync() + { + // Invoking --help returns an error code of 1. + var helpOutput = await _utility.ExecuteAzmcpAsync(_azureMcp, "--help", checkErrorCode: false); + + // Parse the help output by looking for "Description". The following line + // is the server name. + // Example: + // + // Description: + // Azure MCP Server + var lines = Regex.Split(helpOutput, "\r\n|\r|\n"); + var isFound = false; + var serverName = string.Empty; + for (int i = 0; i < lines.Length; i++) + { + string? line = lines[i]; + if (line.StartsWith("Description:")) + { + isFound = true; + serverName = lines[i + 1].Trim(); + } + } + + if (!isFound) + { + _logger.LogError("Could not find server name in --help output."); + return null; + } + else + { + var modified = serverName.Replace(' ', '.'); + _logger.LogInformation("Found server name: {ServerName} from help. Using: {ModifiedServerName}", serverName, modified); + + return modified; + } } } From c6bcca419edc0b4a208306f8bdb74003c56e1908 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 12:31:48 -0800 Subject: [PATCH 42/53] Change utility classes into a class --- eng/tools/ToolMetadataExporter/src/Utility.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/src/Utility.cs b/eng/tools/ToolMetadataExporter/src/Utility.cs index 4cf7095ef8..8e5c2aecc3 100644 --- a/eng/tools/ToolMetadataExporter/src/Utility.cs +++ b/eng/tools/ToolMetadataExporter/src/Utility.cs @@ -7,9 +7,9 @@ namespace ToolMetadataExporter; -internal class Utility +public class Utility { - internal static async Task LoadToolsDynamicallyAsync(string serverFile, string workDirectory, bool isCiMode = false) + internal async Task LoadToolsDynamicallyAsync(string serverFile, string workDirectory, bool isCiMode = false) { try { @@ -50,7 +50,7 @@ internal class Utility } } - internal static async Task GetServerName(string serverFile) + internal async Task GetServerName(string serverFile) { var output = await ExecuteAzmcpAsync(serverFile, "--help", checkErrorCode: false); @@ -67,13 +67,13 @@ internal static async Task GetServerName(string serverFile) throw new InvalidOperationException("Could not find server name"); } - internal static async Task GetVersionAsync(string serverFile) + internal async Task GetVersionAsync(string serverFile) { var output = await ExecuteAzmcpAsync(serverFile, "--version", checkErrorCode: false); return output.Trim(); } - internal static string FindAzmcpAsync(string repositoryRoot, bool isCiMode = false) + internal string FindAzmcpAsync(string repositoryRoot, bool isCiMode = false) { var searchRoots = new List { @@ -117,7 +117,7 @@ internal static string FindAzmcpAsync(string repositoryRoot, bool isCiMode = fal throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); } - internal static async Task ExecuteAzmcpAsync(string serverFile, string arguments, + internal async Task ExecuteAzmcpAsync(string serverFile, string arguments, bool isCiMode = false, bool checkErrorCode = true) { var fileInfo = new FileInfo(serverFile); From d8e65c9b3e6aaad1b4bc263b72eaa25060ba5633 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 12:32:18 -0800 Subject: [PATCH 43/53] - Add utility into DI container - Fix Configuration binding for AppConfig. --- eng/tools/ToolMetadataExporter/src/Program.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/src/Program.cs b/eng/tools/ToolMetadataExporter/src/Program.cs index 765bab3d1d..694086235a 100644 --- a/eng/tools/ToolMetadataExporter/src/Program.cs +++ b/eng/tools/ToolMetadataExporter/src/Program.cs @@ -40,6 +40,7 @@ private static void ConfigureServices(IServiceCollection services, IConfiguratio }); services.AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton(); @@ -47,7 +48,7 @@ private static void ConfigureServices(IServiceCollection services, IConfiguratio .Bind(configuration); services.AddOptions() - .Bind(configuration) + .Bind(configuration.GetSection("AppConfig")) .Configure>((existing, commandLineOptions) => { // Command-line IsDryRun overrides appsettings.json file value. From d7bc8cda98f534adfe99b54dee8889760c1a8b02 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 14:43:31 -0800 Subject: [PATCH 44/53] Use lazy initialization. --- .../src/Services/AzmcpProgram.cs | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs index 3eee269556..e89f8edf1a 100644 --- a/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs @@ -20,10 +20,10 @@ public class AzmcpProgram private readonly ILogger _logger; private readonly Task _serverInfoTask; private readonly Task _serverNameTask; - private readonly Task _listToolsTask; + private readonly Lazy> _listToolsTask; private readonly Task _serverVersionTask; - public AzmcpProgram(IOptions options, Utility utility, ILogger logger) + public AzmcpProgram(Utility utility, IOptions options, ILogger logger) { _toolDirectory = options.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); @@ -35,7 +35,7 @@ public AzmcpProgram(IOptions options, Utility utility, ILogger _serverInfoTask = GetServerInfoInternalAsync(); _serverNameTask = GetServerNameInternalAsync(); _serverVersionTask = GetServerVersionInternalAsync(); - _listToolsTask = GetServerToolsInternalAsync(); + _listToolsTask = new Lazy>(() =>GetServerToolsInternalAsync()); } /// @@ -54,7 +54,7 @@ public AzmcpProgram(IOptions options, Utility utility, ILogger /// Gets the list of tools from the MCP server. /// /// - public virtual Task LoadToolsDynamicallyAsync() => _listToolsTask; + public virtual Task LoadToolsDynamicallyAsync() => _listToolsTask.Value; /// /// Gets the server name of the MCP server in lower-case @@ -115,14 +115,21 @@ private async Task GetServerVersionInternalAsync() { var output = await _utility.ExecuteAzmcpAsync(_azureMcp, "server info", checkErrorCode: false); - var result = JsonSerializer.Deserialize(output, ModelsSerializationContext.Default.ServerInfoResult); + try + { + var result = JsonSerializer.Deserialize(output, ModelsSerializationContext.Default.ServerInfoResult); + if (result == null || result.Results == null) + { + _logger.LogInformation("The MCP server returned an invalid JSON response. Output: {Output}", output); + } - if (result == null || result.Results == null) + return result?.Results; + } + catch (JsonException ex) { - _logger.LogInformation("The MCP server did not return valid JSON output for the 'server info' command. Output: {Output}", output); + _logger.LogInformation(ex, "The MCP server did not return valid JSON output for the 'server info' command. Output: {Output}", output); + return null; } - - return result?.Results; } /// @@ -132,11 +139,11 @@ private async Task GetServerVersionInternalAsync() private async Task InvokeServerVersionCommandAsync() { // Invoking --version returns an error code of 1. - var versionOutput = await _utility.ExecuteAzmcpAsync(_azureMcp, "--version", checkErrorCode: false); + var versionOutput = (await _utility.ExecuteAzmcpAsync(_azureMcp, "--version", checkErrorCode: false)).Trim(); // The version output may contain a git hash after a '+' character. // Example: "1.0.0+4c6c98bca777f54350e426c01177a2b91ad12fd4" - int hashSeparator = versionOutput.Trim().IndexOf('+'); + int hashSeparator = versionOutput.IndexOf('+'); if (hashSeparator != -1) { versionOutput = versionOutput.Substring(0, hashSeparator); From 0249c0be77661ce9cf22d7171be8de819faa40aa Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 14:43:38 -0800 Subject: [PATCH 45/53] Add tests --- .../Services/AzmcpProgramTests.cs | 379 ++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzmcpProgramTests.cs diff --git a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzmcpProgramTests.cs b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzmcpProgramTests.cs new file mode 100644 index 0000000000..3d791ff44d --- /dev/null +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzmcpProgramTests.cs @@ -0,0 +1,379 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Services; +using Xunit; + +namespace ToolMetadataExporter.UnitTests.Services; + +public class AzmcpProgramTests +{ + private readonly Utility _utility; + private readonly ILogger _logger; + private readonly IOptions _options; + private readonly AppConfiguration _appConfiguration; + + public AzmcpProgramTests() + { + _utility = Substitute.For(); + _logger = Substitute.For>(); + _options = Substitute.For>(); + + _appConfiguration = new AppConfiguration + { + AzmcpExe = "azmcp", + WorkDirectory = "/tmp" + }; + + _options.Value.Returns(_appConfiguration); + } + + /// + /// Verifies that the server name is correctly retrieved from the server information response and is lowercase. + /// + [Fact] + public async Task GetsServerNameWithServerInfo() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Template.Mcp.Server", Version = "1.0.0-beta.1+20-0-2" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerNameAsync(); + + Assert.Equal(serverInfo.Name.ToLowerInvariant(), actual); + } + + /// + /// Verifies that the server name is correctly retrieved from the help command output when server info fails. + /// Checks that the name is formatted to lowercase with dots replacing spaces. + /// + [Fact] + public async Task GetsServerNameWithHelp() + { + // Arrange + var serverInfoOutput = + """ + Required command was not provided. + Unrecognized command or argument 'info'. + + Description: + MCP Server operations - Commands for managing and interacting with the MCP Server. + + Usage: + azmcp server [command] [options] + + Options: + -?, -h, --help Show help and usage information + """; + var helpOutput = """ + Description: + Template MCP Server + + Usage: + azmcp [command] [options] + + Options: + -?, -h, --help Show help and usage information + --version Show version information + + Commands: + get_bestpractices Azure best practices - Commands return a list of best practices for code generation, + """; + + _utility.ExecuteAzmcpAsync(Arg.Any(), "server info", checkErrorCode: false).Returns(serverInfoOutput); + _utility.ExecuteAzmcpAsync(Arg.Any(), "--help", checkErrorCode: false).Returns(helpOutput); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerNameAsync(); + + Assert.Equal("template.mcp.server", actual); + } + + [Fact] + public async Task GetsServerVersionFromServerInfo() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Azure.Mcp.Server", Version = "1.2.3-beta.4" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("1.2.3-beta.4", actual); + } + + [Fact] + public async Task GetsServerVersionFromVersionCommand() + { + // Arrange + var serverInfoOutput = "invalid json"; + var versionOutput = "2.0.1"; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serverInfoOutput)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--version", checkErrorCode: false).Returns(Task.FromResult(versionOutput)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("2.0.1", actual); + } + + [Fact] + public async Task GetsServerVersionStripsGitHash() + { + // Arrange + var serverInfoOutput = "invalid json"; + var versionOutput = "1.5.0+abc123def456"; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serverInfoOutput)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--version", checkErrorCode: false).Returns(Task.FromResult(versionOutput)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("1.5.0", actual); + } + + [Fact] + public async Task LoadToolsDynamicallyAsync_CallsUtilityMethod() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Test.Server", Version = "1.0.0" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.LoadToolsDynamicallyAsync(); + + // Assert - Verify the utility method was called + await _utility.Received(1).LoadToolsDynamicallyAsync(_appConfiguration.AzmcpExe!, _appConfiguration.WorkDirectory!, false); + } + + [Fact] + public void Constructor_ThrowsWhenWorkDirectoryIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + AzmcpExe = "azmcp", + WorkDirectory = null + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => new AzmcpProgram(_utility, invalidOptions, _logger)); + } + + [Fact] + public void Constructor_ThrowsWhenAzmcpExeIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + AzmcpExe = null, + WorkDirectory = "/tmp" + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => new AzmcpProgram(_utility, invalidOptions, _logger)); + } + + [Fact] + public async Task GetServerNameAsync_ThrowsWhenBothServerInfoAndHelpFail() + { + // Arrange + var serverInfoOutput = "invalid json"; + var helpOutput = """ + Usage: + azmcp [command] [options] + + Options: + -?, -h, --help Show help and usage information + """; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serverInfoOutput)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--help", checkErrorCode: false).Returns(Task.FromResult(helpOutput)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => await program.GetServerNameAsync()); + Assert.Contains("Failed to determine server name", exception.Message); + } + + [Fact] + public async Task GetServerInfoInternal_ReturnsNullForNullResults() + { + // Arrange + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = null + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + var helpOutput = """ + Description: + Null Results Server + + Usage: + azmcp [command] [options] + """; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--help", checkErrorCode: false).Returns(Task.FromResult(helpOutput)); + + // Act + var program = new AzmcpProgram(_utility, _options, _logger); + var actual = await program.GetServerNameAsync(); + + // Assert + Assert.Equal("null.results.server", actual); + } + + [Fact] + public async Task GetServerVersionAsync_HandlesVersionWithWhitespace() + { + // Arrange + var serverInfoOutput = "invalid json"; + var versionOutput = " 3.2.1 \n"; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serverInfoOutput)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--version", checkErrorCode: false).Returns(Task.FromResult(versionOutput)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("3.2.1", actual); + } + + [Fact] + public async Task GetServerNameAsync_CachesResult() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Cached.Server", Version = "1.0.0" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act - Call twice + var firstResult = await program.GetServerNameAsync(); + var secondResult = await program.GetServerNameAsync(); + + // Assert + Assert.Equal("cached.server", firstResult); + Assert.Equal(firstResult, secondResult); + + // Verify ExecuteAzmcpAsync was only called once for server info (during construction) + await _utility.Received(1).ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false); + } + + [Fact] + public async Task GetServerVersionAsync_CachesResult() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Test.Server", Version = "5.6.7" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act - Call twice + var firstResult = await program.GetServerVersionAsync(); + var secondResult = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("5.6.7", firstResult); + Assert.Equal(firstResult, secondResult); + + // Verify ExecuteAzmcpAsync was only called once for server info (during construction) + await _utility.Received(1).ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false); + } + + [Fact] + public async Task LoadToolsDynamicallyAsync_UsesLazyInitialization() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Test.Server", Version = "1.0.0" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act - Call twice to verify lazy initialization + var firstResult = await program.LoadToolsDynamicallyAsync(); + var secondResult = await program.LoadToolsDynamicallyAsync(); + + // Assert - Verify LoadToolsDynamicallyAsync was only called once (lazy initialization) + await _utility.Received(1).LoadToolsDynamicallyAsync(_appConfiguration.AzmcpExe!, _appConfiguration.WorkDirectory!, false); + } +} From 0c90ff62ddd2fb003e316e7dc26ee02e7ac53f60 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 14:43:58 -0800 Subject: [PATCH 46/53] Make methods virtual for testability --- eng/tools/ToolMetadataExporter/src/Utility.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/src/Utility.cs b/eng/tools/ToolMetadataExporter/src/Utility.cs index 8e5c2aecc3..629118a27e 100644 --- a/eng/tools/ToolMetadataExporter/src/Utility.cs +++ b/eng/tools/ToolMetadataExporter/src/Utility.cs @@ -9,7 +9,7 @@ namespace ToolMetadataExporter; public class Utility { - internal async Task LoadToolsDynamicallyAsync(string serverFile, string workDirectory, bool isCiMode = false) + internal virtual async Task LoadToolsDynamicallyAsync(string serverFile, string workDirectory, bool isCiMode = false) { try { @@ -117,7 +117,16 @@ internal string FindAzmcpAsync(string repositoryRoot, bool isCiMode = false) throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); } - internal async Task ExecuteAzmcpAsync(string serverFile, string arguments, + /// + /// Invokes the azmcp executable with the specified arguments and returns the standard output. + /// + /// Assembly to invoke. + /// Arguments to program. + /// True if it is in CI mode. + /// True to check error code and throw an InvalidOperationException if code is not 0. + /// Standard output as a string. + /// If is true and exit code is not 0. + internal virtual async Task ExecuteAzmcpAsync(string serverFile, string arguments, bool isCiMode = false, bool checkErrorCode = true) { var fileInfo = new FileInfo(serverFile); From ec744b358d725c4e6aa5f85fbc74e1098bab7cc8 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 15:26:07 -0800 Subject: [PATCH 47/53] Add tests for AzaureMcpKustoDatastore --- .../Services/AzureMcpKustoDatastoreTests.cs | 689 ++++++++++++++++++ 1 file changed, 689 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs diff --git a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs new file mode 100644 index 0000000000..8ffcfc1612 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs @@ -0,0 +1,689 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Data; +using Kusto.Data.Common; +using Kusto.Ingest; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using ToolMetadataExporter.Models.Kusto; +using ToolMetadataExporter.Services; +using Xunit; + +namespace ToolMetadataExporter.UnitTests.Services; + +public class AzureMcpKustoDatastoreTests : IDisposable +{ + private readonly ICslQueryProvider _kustoClient; + private readonly IKustoIngestClient _ingestClient; + private readonly ILogger _logger; + private readonly IOptions _options; + private readonly AppConfiguration _appConfiguration; + private readonly string _tempQueriesDirectory; + + public AzureMcpKustoDatastoreTests() + { + _kustoClient = Substitute.For(); + _ingestClient = Substitute.For(); + _logger = Substitute.For>(); + _options = Substitute.For>(); + + // Create a temporary directory for queries + _tempQueriesDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(_tempQueriesDirectory); + + _appConfiguration = new AppConfiguration + { + DatabaseName = "TestDatabase", + McpToolEventsTableName = "McpToolEvents", + QueriesFolder = _tempQueriesDirectory + }; + + _options.Value.Returns(_appConfiguration); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_tempQueriesDirectory)) + { + Directory.Delete(_tempQueriesDirectory, recursive: true); + } + } + catch (Exception) + { + // Suppress cleanup exceptions to avoid failing tests + // The OS will eventually clean up temp directories + } + } + + [Fact] + public void Constructor_ThrowsWhenDatabaseNameIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + DatabaseName = null, + McpToolEventsTableName = "McpToolEvents", + QueriesFolder = _tempQueriesDirectory + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => + new AzureMcpKustoDatastore(_kustoClient, _ingestClient, invalidOptions, _logger)); + } + + [Fact] + public void Constructor_ThrowsWhenTableNameIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + DatabaseName = "TestDatabase", + McpToolEventsTableName = null, + QueriesFolder = _tempQueriesDirectory + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => + new AzureMcpKustoDatastore(_kustoClient, _ingestClient, invalidOptions, _logger)); + } + + [Fact] + public void Constructor_ThrowsWhenQueriesFolderIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + DatabaseName = "TestDatabase", + McpToolEventsTableName = "McpToolEvents", + QueriesFolder = null + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => + new AzureMcpKustoDatastore(_kustoClient, _ingestClient, invalidOptions, _logger)); + } + + [Fact] + public void Constructor_ThrowsWhenQueriesFolderDoesNotExist() + { + // Arrange + var nonExistentPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var invalidConfig = new AppConfiguration + { + DatabaseName = "TestDatabase", + McpToolEventsTableName = "McpToolEvents", + QueriesFolder = nonExistentPath + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + var exception = Assert.Throws(() => + new AzureMcpKustoDatastore(_kustoClient, _ingestClient, invalidOptions, _logger)); + Assert.Contains("does not exist", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsWhenQueryFileNotFound() + { + // Arrange + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("Could not find GetAvailableTools.kql", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ReturnsCreatedTools() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + var result = await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.Single(result); + Assert.Equal("tool-1", result[0].ToolId); + Assert.Equal("TestTool", result[0].ToolName); + Assert.Equal("TestArea", result[0].ToolArea); + } + + [Fact] + public async Task GetAvailableToolsAsync_ReturnsUpdatedTools() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Updated, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "OldTool", + ToolArea = "OldArea", + ReplacedByToolName = "NewTool", + ReplacedByToolArea = "NewArea" + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + var result = await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.Single(result); + Assert.Equal("tool-1", result[0].ToolId); + Assert.Equal("NewTool", result[0].ToolName); + Assert.Equal("NewArea", result[0].ToolArea); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsForUnsupportedEventType() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Deleted, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("unsupported event type", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsWhenToolIdIsEmpty() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("Cannot have an event with no id", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsWhenToolNameIsEmpty() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("without tool name and/or a tool area", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsWhenToolAreaIsEmpty() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("without tool name and/or a tool area", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ProcessesMultipleTools() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "Tool1", + ToolArea = "Area1", + ReplacedByToolName = null, + ReplacedByToolArea = null + }, + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Updated, + ServerVersion = "1.0.0", + ToolId = "tool-2", + ToolName = "OldTool2", + ToolArea = "OldArea2", + ReplacedByToolName = "NewTool2", + ReplacedByToolArea = "NewArea2" + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + var result = await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.Equal(2, result.Count); + Assert.Equal("tool-1", result[0].ToolId); + Assert.Equal("Tool1", result[0].ToolName); + Assert.Equal("tool-2", result[1].ToolId); + Assert.Equal("NewTool2", result[1].ToolName); + } + + [Fact] + public async Task AddToolEventsAsync_IngestsDataSuccessfully() + { + // Arrange + var toolEvents = new List + { + new() + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }; + + var mockIngestionStatus = Substitute.For(); + var statusCollection = new List + { + new() + { + IngestionSourceId = Guid.NewGuid(), + Table = "McpToolEvents", + Status = Status.Succeeded, + Details = "Success" + } + }; + mockIngestionStatus.GetIngestionStatusCollection().Returns(statusCollection); + + _ingestClient.IngestFromStreamAsync( + Arg.Any(), + Arg.Any()) + .Returns(mockIngestionStatus); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + await datastore.AddToolEventsAsync(toolEvents, TestContext.Current.CancellationToken); + + // Assert + await _ingestClient.Received(1).IngestFromStreamAsync( + Arg.Any(), + Arg.Is(p => + p.DatabaseName == "TestDatabase" && + p.TableName == "McpToolEvents")); + } + + [Fact] + public async Task AddToolEventsAsync_LogsWarningWhenResultIsNull() + { + // Arrange + var toolEvents = new List + { + new() + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }; + + _ingestClient.IngestFromStreamAsync( + Arg.Any(), + Arg.Any()) + .Returns((IKustoIngestionResult?)null); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + await datastore.AddToolEventsAsync(toolEvents, TestContext.Current.CancellationToken); + + // Assert + await _ingestClient.Received(1).IngestFromStreamAsync( + Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task AddToolEventsAsync_HandlesEmptyList() + { + // Arrange + var toolEvents = new List(); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + await datastore.AddToolEventsAsync(toolEvents, TestContext.Current.CancellationToken); + + // Assert + await _ingestClient.Received(1).IngestFromStreamAsync( + Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task GetLatestToolEventsAsync_ThrowsWhenKqlFileNotFound() + { + // Arrange + var nonExistentFile = Path.Combine(_tempQueriesDirectory, "NonExistent.kql"); + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in datastore.GetLatestToolEventsAsync(nonExistentFile, TestContext.Current.CancellationToken)) + { + // Should not reach here + } + }); + } + + [Fact] + public async Task GetLatestToolEventsAsync_ReadsEventsFromKusto() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "TestQuery.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + var results = new List(); + await foreach (var toolEvent in datastore.GetLatestToolEventsAsync(queryFile, TestContext.Current.CancellationToken)) + { + results.Add(toolEvent); + } + + // Assert + Assert.Single(results); + Assert.Equal("tool-1", results[0].ToolId); + Assert.Equal("TestTool", results[0].ToolName); + } + + [Fact] + public async Task GetLatestToolEventsAsync_ThrowsOnInvalidEventType() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "TestQuery.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = Substitute.For(); + mockReader.Read().Returns(true, false); + mockReader.GetOrdinal("EventTime").Returns(0); + mockReader.GetOrdinal("EventType").Returns(1); + mockReader.GetOrdinal("ServerVersion").Returns(2); + mockReader.GetOrdinal("ToolId").Returns(3); + mockReader.GetOrdinal("ToolName").Returns(4); + mockReader.GetOrdinal("ToolArea").Returns(5); + mockReader.GetOrdinal("ReplacedByToolName").Returns(6); + mockReader.GetOrdinal("ReplacedByToolArea").Returns(7); + + mockReader.GetDateTime(0).Returns(DateTime.UtcNow); + mockReader.GetString(1).Returns("InvalidEventType"); + mockReader.GetString(2).Returns("1.0.0"); + mockReader.GetString(3).Returns("tool-1"); + mockReader.GetString(4).Returns("TestTool"); + mockReader.GetString(5).Returns("TestArea"); + mockReader.IsDBNull(6).Returns(true); + mockReader.IsDBNull(7).Returns(true); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in datastore.GetLatestToolEventsAsync(queryFile, TestContext.Current.CancellationToken)) + { + // Should not reach here + } + }); + } + + private static IDataReader CreateMockDataReader(McpToolEvent[] events) + { + var reader = Substitute.For(); + var currentIndex = -1; + + reader.Read().Returns(callInfo => + { + currentIndex++; + return currentIndex < events.Length; + }); + + reader.GetOrdinal("EventTime").Returns(0); + reader.GetOrdinal("EventType").Returns(1); + reader.GetOrdinal("ServerVersion").Returns(2); + reader.GetOrdinal("ToolId").Returns(3); + reader.GetOrdinal("ToolName").Returns(4); + reader.GetOrdinal("ToolArea").Returns(5); + reader.GetOrdinal("ReplacedByToolName").Returns(6); + reader.GetOrdinal("ReplacedByToolArea").Returns(7); + + reader.GetDateTime(Arg.Is(0)).Returns(x => GetCurrentEventTime()); + reader.GetString(Arg.Is(1)).Returns(x => GetCurrentEventType()); + reader.GetString(Arg.Is(2)).Returns(x => GetCurrentServerVersion()); + reader.GetString(Arg.Is(3)).Returns(x => GetCurrentToolId()); + reader.GetString(Arg.Is(4)).Returns(x => GetCurrentToolName()); + reader.GetString(Arg.Is(5)).Returns(x => GetCurrentToolArea()); + + reader.IsDBNull(Arg.Is(6)).Returns(x => IsReplacedByToolNameNull()); + reader.GetString(Arg.Is(6)).Returns(x => GetCurrentReplacedByToolName()); + + reader.IsDBNull(Arg.Is(7)).Returns(x => IsReplacedByToolAreaNull()); + reader.GetString(Arg.Is(7)).Returns(x => GetCurrentReplacedByToolArea()); + + return reader; + + DateTime GetCurrentEventTime() => events[Math.Max(0, currentIndex)].EventTime?.DateTime ?? DateTime.UtcNow; + string GetCurrentEventType() => events[Math.Max(0, currentIndex)].EventType?.ToString() ?? string.Empty; + string GetCurrentServerVersion() => events[Math.Max(0, currentIndex)].ServerVersion ?? string.Empty; + string GetCurrentToolId() => events[Math.Max(0, currentIndex)].ToolId ?? string.Empty; + string GetCurrentToolName() => events[Math.Max(0, currentIndex)].ToolName ?? string.Empty; + string GetCurrentToolArea() => events[Math.Max(0, currentIndex)].ToolArea ?? string.Empty; + bool IsReplacedByToolNameNull() => string.IsNullOrEmpty(events[Math.Max(0, currentIndex)].ReplacedByToolName); + string GetCurrentReplacedByToolName() => events[Math.Max(0, currentIndex)].ReplacedByToolName ?? string.Empty; + bool IsReplacedByToolAreaNull() => string.IsNullOrEmpty(events[Math.Max(0, currentIndex)].ReplacedByToolArea); + string GetCurrentReplacedByToolArea() => events[Math.Max(0, currentIndex)].ReplacedByToolArea ?? string.Empty; + } +} From 4657c4402dd62bc8aeb2f2afe14e376bf698324f Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 15:43:46 -0800 Subject: [PATCH 48/53] Add ToolAnalyzerTests --- .../ToolAnalyzerTests.cs | 575 ++++++++++++++++++ 1 file changed, 575 insertions(+) create mode 100644 eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs diff --git a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs new file mode 100644 index 0000000000..5335b58f93 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs @@ -0,0 +1,575 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; +using ToolMetadataExporter.Services; +using ToolSelection.Models; +using Xunit; + +namespace ToolMetadataExporter.UnitTests; + +public class ToolAnalyzerTests : IDisposable +{ + private readonly AzmcpProgram _azmcpProgram; + private readonly IAzureMcpDatastore _datastore; + private readonly ILogger _logger; + private readonly IOptions _options; + private readonly AppConfiguration _appConfiguration; + private readonly string _tempWorkingDirectory; + + public ToolAnalyzerTests() + { + var utility = Substitute.For(); + var logger = Substitute.For>(); + var programOptions = Substitute.For>(); + var programConfig = new AppConfiguration + { + WorkDirectory = Path.GetTempPath(), + AzmcpExe = "azmcp" + }; + programOptions.Value.Returns(programConfig); + + _azmcpProgram = Substitute.ForPartsOf(utility, programOptions, logger); + _datastore = Substitute.For(); + _logger = Substitute.For>(); + _options = Substitute.For>(); + + // Create a temporary directory for working files + _tempWorkingDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(_tempWorkingDirectory); + + _appConfiguration = new AppConfiguration + { + WorkDirectory = _tempWorkingDirectory, + IsDryRun = false + }; + + _options.Value.Returns(_appConfiguration); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_tempWorkingDirectory)) + { + Directory.Delete(_tempWorkingDirectory, recursive: true); + } + } + catch (Exception) + { + // Suppress cleanup exceptions to avoid failing tests + } + } + + [Fact] + public void Constructor_ThrowsWhenWorkDirectoryIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + WorkDirectory = null, + IsDryRun = false + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => + new ToolAnalyzer(_azmcpProgram, _datastore, invalidOptions, _logger)); + } + + [Fact] + public async Task RunAsync_ReturnsEarly_WhenLoadToolsDynamicallyReturnsNull() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(null)); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().GetAvailableToolsAsync(Arg.Any()); + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_ReturnsEarly_WhenToolsListIsNull() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult { Tools = null })); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().GetAvailableToolsAsync(Arg.Any()); + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_ReturnsEarly_WhenToolsListIsEmpty() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult { Tools = [] })); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().GetAvailableToolsAsync(Arg.Any()); + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_ThrowsException_WhenToolHasNoId() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = null, Name = "test-tool", Command = "area command" } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken)); + Assert.Contains("Tool without an id", exception.Message); + } + + [Fact] + public async Task RunAsync_ThrowsException_WhenToolHasNoCommand() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = "tool-1", Name = "test-tool", Command = null } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken)); + Assert.Contains("Tool without a tool area", exception.Message); + } + + [Fact] + public async Task RunAsync_DetectsNewTool() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var serverName = "test-server"; + var serverVersion = "1.0.0"; + var tool = new Tool { Id = "tool-1", Name = "New Tool", Command = "area command" }; + var expectedToolName = "area_command"; + var expectedToolArea = "area"; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult(serverName)); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult(serverVersion)); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].EventType == McpToolEventType.Created && + events[0].ToolId == tool.Id && + events[0].ToolName == expectedToolName && + events[0].ToolArea == expectedToolArea && + events[0].ServerName == serverName && + events[0].ServerVersion == serverVersion), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_DetectsUpdatedTool() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var serverName = "test-server"; + var serverVersion = "1.0.0"; + var tool = new Tool { Id = "tool-1", Name = "Updated Tool", Command = "newarea newcommand" }; + var existingTool = new AzureMcpTool("tool-1", "oldarea_oldcommand", "oldarea"); + var expectedNewToolName = "newarea_newcommand"; + var expectedNewToolArea = "newarea"; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult(serverName)); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult(serverVersion)); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].EventType == McpToolEventType.Updated && + events[0].ToolId == tool.Id && + events[0].ToolName == existingTool.ToolName && + events[0].ToolArea == existingTool.ToolArea && + events[0].ReplacedByToolName == expectedNewToolName && + events[0].ReplacedByToolArea == expectedNewToolArea), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_DetectsDeletedTool() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var serverName = "test-server"; + var serverVersion = "1.0.0"; + var existingTool = new AzureMcpTool("tool-1", "area_command", "area"); + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult(serverName)); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult(serverVersion)); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].EventType == McpToolEventType.Deleted && + events[0].ToolId == existingTool.ToolId && + events[0].ToolName == existingTool.ToolName && + events[0].ToolArea == existingTool.ToolArea && + events[0].ReplacedByToolName == null && + events[0].ReplacedByToolArea == null), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_DoesNotDetectChange_WhenToolUnchanged() + { + // Arrange + var tool = new Tool { Id = "tool-1", Name = "Test Tool", Command = "area command" }; + var existingTool = new AzureMcpTool("tool-1", "area_command", "area"); + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_HandlesMultipleChanges() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var tool1 = new Tool { Id = "tool-1", Name = "Tool 1", Command = "area1 command1" }; // Unchanged + var tool2 = new Tool { Id = "tool-2", Name = "Tool 2", Command = "area2 newcommand" }; // Updated + var tool4 = new Tool { Id = "tool-4", Name = "Tool 4", Command = "area4 command4" }; // New + var existingTool1 = new AzureMcpTool("tool-1", "area1_command1", "area1"); + var existingTool2 = new AzureMcpTool("tool-2", "area2_oldcommand", "area2"); + var existingTool3 = new AzureMcpTool("tool-3", "area3_command3", "area3"); // Deleted + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool1, tool2, tool4] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool1, + existingTool2, + existingTool3 + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 3 && + events.Any(e => e.EventType == McpToolEventType.Updated && e.ToolId == tool2.Id) && + events.Any(e => e.EventType == McpToolEventType.Created && e.ToolId == tool4.Id) && + events.Any(e => e.EventType == McpToolEventType.Deleted && e.ToolId == existingTool3.ToolId)), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_WritesChangesToFile() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var tool = new Tool { Id = "tool-1", Name = "New Tool", Command = "area command" }; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + var outputFile = Path.Combine(_tempWorkingDirectory, "tool_changes.json"); + Assert.True(File.Exists(outputFile)); + var fileContent = await File.ReadAllTextAsync(outputFile, TestContext.Current.CancellationToken); + Assert.Contains(tool.Id, fileContent); + Assert.Contains("Created", fileContent); + } + + [Fact] + public async Task RunAsync_SkipsDatastoreUpdate_WhenDryRunIsTrue() + { + // Arrange + _appConfiguration.IsDryRun = true; + var analysisTime = DateTimeOffset.UtcNow; + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = "tool-1", Name = "New Tool", Command = "area command" } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + + // But file should still be written + var outputFile = Path.Combine(_tempWorkingDirectory, "tool_changes.json"); + Assert.True(File.Exists(outputFile)); + } + + [Fact] + public async Task RunAsync_HandlesCancellation() + { + // Arrange + var cts = new CancellationTokenSource(); + cts.Cancel(); + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = "tool-1", Name = "Test Tool", Command = "area command" } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, cts.Token); + + // Assert - Should return early without throwing + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_NormalizesCommandToLowercase() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var tool = new Tool { Id = "tool-1", Name = "Test Tool", Command = "Area Command" }; + var expectedToolName = "area_command"; + var expectedToolArea = "area"; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].ToolName == expectedToolName && + events[0].ToolArea == expectedToolArea), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_ReplacesSpacesWithUnderscores() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = "tool-1", Name = "Test Tool", Command = "area command with spaces" } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].ToolName == "area_command_with_spaces"), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_DetectsToolAreaChange() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var tool = new Tool { Id = "tool-1", Name = "Test Tool", Command = "newarea command" }; + var existingTool = new AzureMcpTool("tool-1", "oldarea_command", "oldarea"); + var expectedNewToolArea = "newarea"; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].EventType == McpToolEventType.Updated && + events[0].ToolArea == existingTool.ToolArea && + events[0].ReplacedByToolArea == expectedNewToolArea), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_IsCaseInsensitive_ForComparison() + { + // Arrange + var tool = new Tool { Id = "tool-1", Name = "Test Tool", Command = "AREA COMMAND" }; + var existingTool = new AzureMcpTool("tool-1", "area_command", "area"); + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert - Should not detect any changes + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } +} From c23ca90249c87c2ebb149f6194c0657ab6a80400 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 15:57:43 -0800 Subject: [PATCH 49/53] Add tool projects to solution --- AzureMcp.sln | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/AzureMcp.sln b/AzureMcp.sln index 4910d24c0d..a22f21ec2d 100644 --- a/AzureMcp.sln +++ b/AzureMcp.sln @@ -1,7 +1,6 @@ - Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.31903.59 +# Visual Studio Version 18 +VisualStudioVersion = 18.1.11312.151 d18.0 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "core", "core", "{FBF56CC3-7AE6-AD2D-3F14-7F97FD322CD6}" EndProject @@ -591,6 +590,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{294AC723 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fabric.Mcp.Tools.PublicApi.UnitTests", "tools\Fabric.Mcp.Tools.PublicApi\tests\Fabric.Mcp.Tools.PublicApi.UnitTests\Fabric.Mcp.Tools.PublicApi.UnitTests.csproj", "{D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ToolMetadataExporter", "ToolMetadataExporter", "{BB32CFBE-B28C-4ADD-B518-0B161017E82B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter.UnitTests", "eng\tools\ToolMetadataExporter\tests\ToolMetadataExporter.UnitTests\ToolMetadataExporter.UnitTests.csproj", "{72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter", "eng\tools\ToolMetadataExporter\src\ToolMetadataExporter.csproj", "{7438F013-054C-3F6E-D56C-D6559BFD835E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -2233,6 +2238,30 @@ Global {D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F}.Release|x64.Build.0 = Release|Any CPU {D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F}.Release|x86.ActiveCfg = Release|Any CPU {D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F}.Release|x86.Build.0 = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|x64.ActiveCfg = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|x64.Build.0 = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|x86.ActiveCfg = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|x86.Build.0 = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|Any CPU.Build.0 = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|x64.ActiveCfg = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|x64.Build.0 = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|x86.ActiveCfg = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|x86.Build.0 = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|x64.Build.0 = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|x86.Build.0 = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|Any CPU.Build.0 = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|x64.ActiveCfg = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|x64.Build.0 = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|x86.ActiveCfg = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2528,6 +2557,8 @@ Global {E3F46C2D-3AFD-FD9C-9C6A-180B1514DD30} = {7CB41572-AACD-B631-FA61-D313B4B8C8E9} {294AC723-70DA-F50A-2C7A-AC6C0AEA0A62} = {9072C7AF-9EB2-E481-3974-77957587AC76} {D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F} = {294AC723-70DA-F50A-2C7A-AC6C0AEA0A62} + {BB32CFBE-B28C-4ADD-B518-0B161017E82B} = {DAAE2FFB-70A9-DCEF-23A0-0ABAED0A9720} + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD} = {BB32CFBE-B28C-4ADD-B518-0B161017E82B} + {7438F013-054C-3F6E-D56C-D6559BFD835E} = {BB32CFBE-B28C-4ADD-B518-0B161017E82B} EndGlobalSection EndGlobal - From bcdfa4b2fc3b6aa43d954011a099d49a8a56088a Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 16:02:19 -0800 Subject: [PATCH 50/53] Fix formatting error in AzureMcpKustoDatastore --- .../Services/AzureMcpKustoDatastoreTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs index 8ffcfc1612..a7a1db618c 100644 --- a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs @@ -666,10 +666,10 @@ private static IDataReader CreateMockDataReader(McpToolEvent[] events) reader.GetString(Arg.Is(3)).Returns(x => GetCurrentToolId()); reader.GetString(Arg.Is(4)).Returns(x => GetCurrentToolName()); reader.GetString(Arg.Is(5)).Returns(x => GetCurrentToolArea()); - + reader.IsDBNull(Arg.Is(6)).Returns(x => IsReplacedByToolNameNull()); reader.GetString(Arg.Is(6)).Returns(x => GetCurrentReplacedByToolName()); - + reader.IsDBNull(Arg.Is(7)).Returns(x => IsReplacedByToolAreaNull()); reader.GetString(Arg.Is(7)).Returns(x => GetCurrentReplacedByToolArea()); From f2c0d3814fc49c0f00610c47d88a3c5ebcfad06f Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 16:02:58 -0800 Subject: [PATCH 51/53] Fix formatting error in ToolAnalyzerTests --- .../tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs index 5335b58f93..5b54fe4b30 100644 --- a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs @@ -418,7 +418,7 @@ public async Task RunAsync_SkipsDatastoreUpdate_WhenDryRunIsTrue() // Assert await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); - + // But file should still be written var outputFile = Path.Combine(_tempWorkingDirectory, "tool_changes.json"); Assert.True(File.Exists(outputFile)); From cab9ecf92e4bfd843b576c21388cbfd9d6dbef4e Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 16:03:27 -0800 Subject: [PATCH 52/53] Fix whitespace AzmcpProgram --- eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs index e89f8edf1a..a75637ab5e 100644 --- a/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs +++ b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs @@ -35,7 +35,7 @@ public AzmcpProgram(Utility utility, IOptions options, ILogger _serverInfoTask = GetServerInfoInternalAsync(); _serverNameTask = GetServerNameInternalAsync(); _serverVersionTask = GetServerVersionInternalAsync(); - _listToolsTask = new Lazy>(() =>GetServerToolsInternalAsync()); + _listToolsTask = new Lazy>(() => GetServerToolsInternalAsync()); } /// From d30a9b6da50e51a5e8b5521213d3318b8c43a7f1 Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Mon, 12 Jan 2026 16:13:28 -0800 Subject: [PATCH 53/53] Add Ticks to FileName. --- eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs index 231a4006ee..828a8cff2f 100644 --- a/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs +++ b/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs @@ -143,7 +143,8 @@ public async Task RunAsync(DateTimeOffset analysisTime, CancellationToken cancel return; } - var outputFile = Path.Combine(_workingDirectory, "tool_changes.json"); + var filename = $"tool_changes.{analysisTime.Ticks}.json"; + var outputFile = Path.Combine(_workingDirectory, filename); _logger.LogInformation("Tool updates. Writing output to: {FileName}", outputFile); @@ -167,6 +168,8 @@ public async Task RunAsync(DateTimeOffset analysisTime, CancellationToken cancel } } + cancellationToken.ThrowIfCancellationRequested(); + if (!_isDryRun) { _logger.LogInformation("Updating datastore.");