diff --git a/AzureMcp.sln b/AzureMcp.sln index 4910d24c0d..a22f21ec2d 100644 --- a/AzureMcp.sln +++ b/AzureMcp.sln @@ -1,7 +1,6 @@ - Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.31903.59 +# Visual Studio Version 18 +VisualStudioVersion = 18.1.11312.151 d18.0 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "core", "core", "{FBF56CC3-7AE6-AD2D-3F14-7F97FD322CD6}" EndProject @@ -591,6 +590,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{294AC723 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fabric.Mcp.Tools.PublicApi.UnitTests", "tools\Fabric.Mcp.Tools.PublicApi\tests\Fabric.Mcp.Tools.PublicApi.UnitTests\Fabric.Mcp.Tools.PublicApi.UnitTests.csproj", "{D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ToolMetadataExporter", "ToolMetadataExporter", "{BB32CFBE-B28C-4ADD-B518-0B161017E82B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter.UnitTests", "eng\tools\ToolMetadataExporter\tests\ToolMetadataExporter.UnitTests\ToolMetadataExporter.UnitTests.csproj", "{72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter", "eng\tools\ToolMetadataExporter\src\ToolMetadataExporter.csproj", "{7438F013-054C-3F6E-D56C-D6559BFD835E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -2233,6 +2238,30 @@ Global {D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F}.Release|x64.Build.0 = Release|Any CPU {D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F}.Release|x86.ActiveCfg = Release|Any CPU {D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F}.Release|x86.Build.0 = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|x64.ActiveCfg = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|x64.Build.0 = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|x86.ActiveCfg = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Debug|x86.Build.0 = Debug|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|Any CPU.Build.0 = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|x64.ActiveCfg = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|x64.Build.0 = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|x86.ActiveCfg = Release|Any CPU + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD}.Release|x86.Build.0 = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|x64.Build.0 = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Debug|x86.Build.0 = Debug|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|Any CPU.Build.0 = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|x64.ActiveCfg = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|x64.Build.0 = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|x86.ActiveCfg = Release|Any CPU + {7438F013-054C-3F6E-D56C-D6559BFD835E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2528,6 +2557,8 @@ Global {E3F46C2D-3AFD-FD9C-9C6A-180B1514DD30} = {7CB41572-AACD-B631-FA61-D313B4B8C8E9} {294AC723-70DA-F50A-2C7A-AC6C0AEA0A62} = {9072C7AF-9EB2-E481-3974-77957587AC76} {D3F46C2D-3AFD-FD9C-9C6A-180B1514DD2F} = {294AC723-70DA-F50A-2C7A-AC6C0AEA0A62} + {BB32CFBE-B28C-4ADD-B518-0B161017E82B} = {DAAE2FFB-70A9-DCEF-23A0-0ABAED0A9720} + {72A3BA32-F77D-FCBD-6A2C-1FEE3C3EA1FD} = {BB32CFBE-B28C-4ADD-B518-0B161017E82B} + {7438F013-054C-3F6E-D56C-D6559BFD835E} = {BB32CFBE-B28C-4ADD-B518-0B161017E82B} EndGlobalSection EndGlobal - diff --git a/Directory.Packages.props b/Directory.Packages.props index 132c759586..b6bb1ed400 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -60,7 +60,8 @@ - + + diff --git a/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs b/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs index ad6ce6c59f..781922e64e 100644 --- a/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs +++ b/eng/tools/ToolDescriptionEvaluator/src/Models/McpModels.cs @@ -83,6 +83,9 @@ public class ToolAnnotations // Tool definition for azmcp tools list response public class Tool { + [JsonPropertyName("id")] + public string? Id { get; set; } + [JsonPropertyName("name")] public required string Name { get; set; } diff --git a/eng/tools/ToolMetadataExporter/src/AppConfiguration.cs b/eng/tools/ToolMetadataExporter/src/AppConfiguration.cs new file mode 100644 index 0000000000..921fe6de18 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/AppConfiguration.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter; + +public class AppConfiguration +{ + public string? IngestionEndpoint { get; set; } + + public string? QueryEndpoint { get; set; } + + public string? DatabaseName { get; set; } + + public string? McpToolEventsTableName { get; set; } + + public string? QueriesFolder { get; set; } = "Resources/queries"; + + public string? WorkDirectory { get; set; } + + public bool IsDryRun { get; set; } + + public string? AzmcpExe { get; set; } + + public bool IsAzmcpExeSpecified { get; set; } +} diff --git a/eng/tools/ToolMetadataExporter/src/AssemblyInfo.cs b/eng/tools/ToolMetadataExporter/src/AssemblyInfo.cs new file mode 100644 index 0000000000..e0e0f77159 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/AssemblyInfo.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ToolMetadataExporter.UnitTests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/eng/tools/ToolMetadataExporter/src/GlobalUsings.cs b/eng/tools/ToolMetadataExporter/src/GlobalUsings.cs new file mode 100644 index 0000000000..38bac6b955 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/GlobalUsings.cs @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +global using System; +global using System.Text.Json; diff --git a/eng/tools/ToolMetadataExporter/src/Models/AzureMcpTool.cs b/eng/tools/ToolMetadataExporter/src/Models/AzureMcpTool.cs new file mode 100644 index 0000000000..26dfecd3f6 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Models/AzureMcpTool.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter.Models; + +public record AzureMcpTool( + string ToolId, + string ToolName, + string ToolArea); diff --git a/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs b/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs new file mode 100644 index 0000000000..a5123f8cb6 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Models/CommandLineOptions.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace ToolMetadataExporter.Models; + +/// +/// Options specified via command line arguments. Supported options are: +/// +/// --dry-run: If specified, the tool will run in dry-run mode, meaning no changes will be made to the target datastore. +/// --azmcp-exe <path>: The path to the azmcp executable to use for interacting with the MCP server. +/// +/// +internal class CommandLineOptions +{ + /// + /// Gets or sets a value indicating whether the tool analysis should be performed as a dry run. + /// + /// When set to , the operation is performed, output to the console, but not persisted to the datastore. When set to + /// , the operation is executed normally. + public bool? IsDryRun { get; set; } + + /// + /// Gets or sets the full path to the AzMcp executable file. + /// + public string? AzmcpExe { get; set; } +} diff --git a/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs b/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs new file mode 100644 index 0000000000..2c2f27dbc2 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEvent.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using Kusto.Data.Common; + +namespace ToolMetadataExporter.Models.Kusto; + +public class McpToolEvent +{ + private const string EventTimeColumn = "EventTime"; + private const string EventTypeColumn = "EventType"; + private const string ServerNameColumn = "ServerName"; + private const string ServerVersionColumn = "ServerVersion"; + private const string ToolIdColumn = "ToolId"; + private const string ToolNameColumn = "ToolName"; + private const string ToolAreaColumn = "ToolArea"; + private const string ReplacedByToolNameColumn = "ReplacedByToolName"; + private const string ReplacedByToolAreaColumn = "ReplacedByToolArea"; + + [JsonPropertyName(EventTimeColumn)] + public DateTimeOffset? EventTime { get; set; } + + [JsonPropertyName(EventTypeColumn)] + public McpToolEventType? EventType { get; set; } + + [JsonPropertyName(ServerNameColumn)] + public string? ServerName { get; set; } + + [JsonPropertyName(ServerVersionColumn)] + public string? ServerVersion { get; set; } + + [JsonPropertyName(ToolIdColumn)] + public string? ToolId { get; set; } + + [JsonPropertyName(ToolNameColumn)] + public string? ToolName { get; set; } + + [JsonPropertyName(ToolAreaColumn)] + public string? ToolArea { get; set; } + + [JsonPropertyName(ReplacedByToolNameColumn)] + public string? ReplacedByToolName { get; set; } + + [JsonPropertyName(ReplacedByToolAreaColumn)] + public string? ReplacedByToolArea { get; set; } + + public static ColumnMapping[] GetColumnMappings() + { + return [ + new ColumnMapping { ColumnName = EventTimeColumn, ColumnType = "datetime" }, + new ColumnMapping { ColumnName = EventTypeColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ReplacedByToolAreaColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ReplacedByToolNameColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ServerVersionColumn, ColumnType = "string" }, + new ColumnMapping { ColumnName = ToolAreaColumn , ColumnType = "string" }, + new ColumnMapping { ColumnName = ToolIdColumn, ColumnType = "string"}, + new ColumnMapping { ColumnName = ToolNameColumn, ColumnType = "string" }, + new ColumnMapping { ColumnName = ServerNameColumn, ColumnType = "string" }, + ]; + } +} diff --git a/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEventType.cs b/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEventType.cs new file mode 100644 index 0000000000..6bc1ea3c54 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Models/Kusto/McpToolEventType.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace ToolMetadataExporter.Models.Kusto; + +public enum McpToolEventType +{ + [JsonStringEnumMemberName("Created")] + Created, + [JsonStringEnumMemberName("Updated")] + Updated, + [JsonStringEnumMemberName("Deleted")] + Deleted +} diff --git a/eng/tools/ToolMetadataExporter/src/Models/ModelsSerializationContext.cs b/eng/tools/ToolMetadataExporter/src/Models/ModelsSerializationContext.cs new file mode 100644 index 0000000000..ac06218715 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Models/ModelsSerializationContext.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; +using ToolMetadataExporter.Models.Kusto; + +namespace ToolMetadataExporter.Models; + +[JsonSerializable(typeof(ServerInfo))] +[JsonSerializable(typeof(ServerInfoResult))] +[JsonSerializable(typeof(McpToolEvent))] +[JsonSerializable(typeof(McpToolEventType))] +[JsonSerializable(typeof(List))] +[JsonSourceGenerationOptions(Converters = [typeof(JsonStringEnumConverter)])] +public partial class ModelsSerializationContext : JsonSerializerContext +{ +} diff --git a/eng/tools/ToolMetadataExporter/src/Models/ServerInfoResult.cs b/eng/tools/ToolMetadataExporter/src/Models/ServerInfoResult.cs new file mode 100644 index 0000000000..ba90e568b5 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Models/ServerInfoResult.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json.Serialization; + +namespace ToolMetadataExporter.Models; + +/// +/// The result of a `server info` request from the MCP server. +/// +public class ServerInfoResult +{ + [JsonPropertyName("status")] + + public int Status { get; set; } + + [JsonPropertyName("message")] + public string? Message { get; set; } + + [JsonPropertyName("results")] + public ServerInfo? Results { get; set; } +} + +public class ServerInfo +{ + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("version")] + public string Version { get; set; } = string.Empty; +} diff --git a/eng/tools/ToolMetadataExporter/src/Program.cs b/eng/tools/ToolMetadataExporter/src/Program.cs new file mode 100644 index 0000000000..694086235a --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Program.cs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Core; +using Azure.Identity; +using Kusto.Data; +using Kusto.Data.Common; +using Kusto.Data.Net.Client; +using Kusto.Ingest; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Services; + +namespace ToolMetadataExporter; + +public class Program +{ + public static async Task Main(string[] args) + { + var builder = Host.CreateApplicationBuilder(args); + + ConfigureServices(builder.Services, builder.Configuration); + ConfigureAzureServices(builder.Services); + + var host = builder.Build(); + var analyzer = host.Services.GetRequiredService(); + + await analyzer.RunAsync(DateTimeOffset.UtcNow); + } + + private static void ConfigureServices(IServiceCollection services, IConfiguration configuration) + { + services.AddLogging(builder => + { + builder.AddConsole(); + }); + + services.AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(); + + services.AddOptions() + .Bind(configuration); + + services.AddOptions() + .Bind(configuration.GetSection("AppConfig")) + .Configure>((existing, commandLineOptions) => + { + // Command-line IsDryRun overrides appsettings.json file value. + if (commandLineOptions.Value.IsDryRun.HasValue) + { + existing.IsDryRun = commandLineOptions.Value.IsDryRun.Value; + } + + var exeDir = AppContext.BaseDirectory; + + // If a path to azmcp.exe is not provided. Assume that this is running within the context of + // the repository and try to find it. + existing.IsAzmcpExeSpecified = !string.IsNullOrEmpty(commandLineOptions.Value.AzmcpExe); + if (existing.IsAzmcpExeSpecified) + { + existing.AzmcpExe = commandLineOptions.Value.AzmcpExe!; + + if (existing.WorkDirectory == null) + { + existing.WorkDirectory = exeDir; + } + } + else + { + var repoRoot = Utility.FindRepoRoot(exeDir); + if (existing.WorkDirectory == null) + { + existing.WorkDirectory = Path.Combine(repoRoot, ".work"); + } + + existing.AzmcpExe = Path.Combine(repoRoot, "eng", "tools", "Azmcp", "azmcp.exe"); + } + }); + } + + private static void ConfigureAzureServices(IServiceCollection services) + { + services.AddScoped(sp => + { + var credential = new ChainedTokenCredential( + new ManagedIdentityCredential(), + new DefaultAzureCredential() + ); + + return credential; + }); + services.AddSingleton(sp => + { + var config = sp.GetRequiredService>(); + + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.QueryEndpoint) + .WithAadUserPromptAuthentication() + .WithAadAzCliAuthentication(interactive: true); + + return KustoClientFactory.CreateCslQueryProvider(connectionStringBuilder); + }); + services.AddSingleton(sp => + { + var config = sp.GetRequiredService>(); + + var connectionStringBuilder = new KustoConnectionStringBuilder(config.Value.IngestionEndpoint) + .WithAadUserPromptAuthentication() + .WithAadAzCliAuthentication(interactive: true); + + return KustoIngestFactory.CreateDirectIngestClient(connectionStringBuilder); + }); + } +} diff --git a/eng/tools/ToolMetadataExporter/src/Properties/launchSettings.json b/eng/tools/ToolMetadataExporter/src/Properties/launchSettings.json new file mode 100644 index 0000000000..7f48f76e44 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Properties/launchSettings.json @@ -0,0 +1,10 @@ +{ + "profiles": { + "ToolMetadataExporter": { + "commandName": "Project", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/eng/tools/ToolMetadataExporter/src/Resources/queries/CreateTable.kql b/eng/tools/ToolMetadataExporter/src/Resources/queries/CreateTable.kql new file mode 100644 index 0000000000..74a14c1c0d --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Resources/queries/CreateTable.kql @@ -0,0 +1,10 @@ +.create table McpToolEvents ( + EventTime: datetime, + EventType: string, + ServerVersion: string, + ToolId: string, + ToolName: string, + ToolArea: string, + ReplacedByToolName: string, + ReplacedByToolArea: string +) diff --git a/eng/tools/ToolMetadataExporter/src/Resources/queries/GetAvailableTools.kql b/eng/tools/ToolMetadataExporter/src/Resources/queries/GetAvailableTools.kql new file mode 100644 index 0000000000..26a901adc8 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Resources/queries/GetAvailableTools.kql @@ -0,0 +1,4 @@ +McpToolEvents + | summarize arg_max(EventTime, *) by ToolId + | where EventType != 'Deleted' + | project EventTime, ToolId, ToolName, ToolArea, ServerVersion, EventType, ReplacedByToolName, ReplacedByToolArea diff --git a/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs new file mode 100644 index 0000000000..a75637ab5e --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Services/AzmcpProgram.cs @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; +using ToolSelection.Models; + +namespace ToolMetadataExporter.Services; + +/// +/// Represents the MCP server and exposes methods to interact with it. +/// +public class AzmcpProgram +{ + private readonly string _toolDirectory; + private readonly string _azureMcp; + private readonly Utility _utility; + private readonly ILogger _logger; + private readonly Task _serverInfoTask; + private readonly Task _serverNameTask; + private readonly Lazy> _listToolsTask; + private readonly Task _serverVersionTask; + + public AzmcpProgram(Utility utility, IOptions options, ILogger logger) + { + _toolDirectory = options.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); + + _azureMcp = options.Value.AzmcpExe + ?? throw new ArgumentNullException(nameof(CommandLineOptions.AzmcpExe)); + _utility = utility; + _logger = logger; + + _serverInfoTask = GetServerInfoInternalAsync(); + _serverNameTask = GetServerNameInternalAsync(); + _serverVersionTask = GetServerVersionInternalAsync(); + _listToolsTask = new Lazy>(() => GetServerToolsInternalAsync()); + } + + /// + /// Gets the name of the MCP server in lower case. + /// + /// + public virtual Task GetServerNameAsync() => _serverNameTask; + + /// + /// Gets the server version. + /// + /// + public virtual Task GetServerVersionAsync() => _serverVersionTask; + + /// + /// Gets the list of tools from the MCP server. + /// + /// + public virtual Task LoadToolsDynamicallyAsync() => _listToolsTask.Value; + + /// + /// Gets the server name of the MCP server in lower-case + /// + /// The server name of the MCP server in lowercase. + /// If name could not be determined. + private async Task GetServerNameInternalAsync() + { + var output = await _serverInfoTask; + + if (output != null) + { + return output.Name.ToLowerInvariant(); + } + + var serverName = await GetServerNameWithHelpCommandAsync(); + + if (serverName == null) + { + throw new InvalidOperationException("Failed to determine server name via `server info` and `--help`."); + } + else + { + return serverName.ToLowerInvariant(); + } + } + + /// + /// Gets the server version of the MCP server. + /// + /// Semver version of the MCP server. + /// If a version could not be determined. + private async Task GetServerVersionInternalAsync() + { + var output = await _serverInfoTask; + + if (output != null) + { + return output.Version; + } + else + { + return await InvokeServerVersionCommandAsync(); + } + } + + private Task GetServerToolsInternalAsync() + { + return _utility.LoadToolsDynamicallyAsync(_azureMcp, _toolDirectory, false); + } + + /// + /// Gets server information by invoking the "server info" command in the MCP server. + /// + /// The server information. Null, if output from `server info` could not be parsed from JSON. + /// This may be the case when "server info" has not been implemented by the server. + private async Task GetServerInfoInternalAsync() + { + var output = await _utility.ExecuteAzmcpAsync(_azureMcp, "server info", checkErrorCode: false); + + try + { + var result = JsonSerializer.Deserialize(output, ModelsSerializationContext.Default.ServerInfoResult); + if (result == null || result.Results == null) + { + _logger.LogInformation("The MCP server returned an invalid JSON response. Output: {Output}", output); + } + + return result?.Results; + } + catch (JsonException ex) + { + _logger.LogInformation(ex, "The MCP server did not return valid JSON output for the 'server info' command. Output: {Output}", output); + return null; + } + } + + /// + /// Invokes the MCP server with the --version argument to get the server version. + /// + /// A semver compatible version. + private async Task InvokeServerVersionCommandAsync() + { + // Invoking --version returns an error code of 1. + var versionOutput = (await _utility.ExecuteAzmcpAsync(_azureMcp, "--version", checkErrorCode: false)).Trim(); + + // The version output may contain a git hash after a '+' character. + // Example: "1.0.0+4c6c98bca777f54350e426c01177a2b91ad12fd4" + int hashSeparator = versionOutput.IndexOf('+'); + if (hashSeparator != -1) + { + versionOutput = versionOutput.Substring(0, hashSeparator); + } + + return versionOutput; + } + + /// + /// Invokes the MCP server with the --help argument to get the server name. + /// + /// The server name for the MCP server. Null if the server name could not be found. + /// When --help is invoked the name is found after the "Description:" line. Spaces in the server + /// name are replaced with periods. Example: "Azure Mcp Server" becomes "Azure.Mcp.Server". + private async Task GetServerNameWithHelpCommandAsync() + { + // Invoking --help returns an error code of 1. + var helpOutput = await _utility.ExecuteAzmcpAsync(_azureMcp, "--help", checkErrorCode: false); + + // Parse the help output by looking for "Description". The following line + // is the server name. + // Example: + // + // Description: + // Azure MCP Server + var lines = Regex.Split(helpOutput, "\r\n|\r|\n"); + var isFound = false; + var serverName = string.Empty; + for (int i = 0; i < lines.Length; i++) + { + string? line = lines[i]; + if (line.StartsWith("Description:")) + { + isFound = true; + serverName = lines[i + 1].Trim(); + } + } + + if (!isFound) + { + _logger.LogError("Could not find server name in --help output."); + return null; + } + else + { + var modified = serverName.Replace(' ', '.'); + _logger.LogInformation("Found server name: {ServerName} from help. Using: {ModifiedServerName}", serverName, modified); + + return modified; + } + } +} diff --git a/eng/tools/ToolMetadataExporter/src/Services/AzureMcpKustoDatastore.cs b/eng/tools/ToolMetadataExporter/src/Services/AzureMcpKustoDatastore.cs new file mode 100644 index 0000000000..5cb7cf42a5 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Services/AzureMcpKustoDatastore.cs @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.CompilerServices; +using Kusto.Data.Common; +using Kusto.Data.Ingestion; +using Kusto.Ingest; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; + +namespace ToolMetadataExporter.Services; + +public class AzureMcpKustoDatastore : IAzureMcpDatastore +{ + private readonly ICslQueryProvider _kustoClient; + private readonly IKustoIngestClient _ingestClient; + private readonly ILogger _logger; + private readonly DirectoryInfo _queriesDirectory; + private readonly string _databaseName; + private readonly string _tableName; + + public AzureMcpKustoDatastore( + ICslQueryProvider kustoClient, + IKustoIngestClient ingestClient, + IOptions configuration, + ILogger logger) + { + _kustoClient = kustoClient; + _ingestClient = ingestClient; + _logger = logger; + + _databaseName = configuration.Value.DatabaseName ?? throw new ArgumentNullException(nameof(AppConfiguration.DatabaseName)); + _tableName = configuration.Value.McpToolEventsTableName ?? throw new ArgumentNullException(nameof(AppConfiguration.McpToolEventsTableName)); + _queriesDirectory = configuration.Value.QueriesFolder == null + ? throw new ArgumentNullException(nameof(configuration.Value.QueriesFolder)) + : new DirectoryInfo(configuration.Value.QueriesFolder); + + if (!_queriesDirectory.Exists) + { + throw new ArgumentException($"'{_queriesDirectory.FullName}' does not exist. Value: {configuration.Value.QueriesFolder}"); + } + } + + public async Task> GetAvailableToolsAsync(CancellationToken cancellationToken = default) + { + var queryFile = _queriesDirectory.GetFiles("GetAvailableTools.kql").FirstOrDefault(); + + if (queryFile == null) + { + throw new InvalidOperationException($"Could not find GetAvailableTools.kql in {_queriesDirectory.FullName}"); + } + + var results = new List(); + + await foreach (var latestEvent in GetLatestToolEventsAsync(queryFile.FullName, cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrEmpty(latestEvent.ToolId)) + { + throw new InvalidOperationException( + $"Cannot have an event with no id. Name: {latestEvent.ToolArea}, Area: {latestEvent.ToolArea}"); + } + + string? toolName; + string? toolArea; + switch (latestEvent.EventType) + { + case McpToolEventType.Created: + toolName = latestEvent.ToolName; + toolArea = latestEvent.ToolArea; + break; + case McpToolEventType.Updated: + toolName = latestEvent.ReplacedByToolName; + toolArea = latestEvent.ReplacedByToolArea; + break; + default: + throw new InvalidOperationException($"Tool '{latestEvent.ToolId}' has unsupported event type: {latestEvent.EventType}"); + } + + if (string.IsNullOrEmpty(toolName) || string.IsNullOrEmpty(toolArea)) + { + throw new InvalidOperationException($"Tool '{latestEvent.ToolId}' without tool name and/or a tool area."); + } + + results.Add(new AzureMcpTool(latestEvent.ToolId, toolName, toolArea)); + } + + return results; + } + + public async Task AddToolEventsAsync(List toolEvents, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + using MemoryStream stream = new MemoryStream(); + + await JsonSerializer.SerializeAsync(stream, toolEvents, ModelsSerializationContext.Default.ListMcpToolEvent, cancellationToken); + stream.Seek(0, SeekOrigin.Begin); + + cancellationToken.ThrowIfCancellationRequested(); + + var ingestionProperties = new KustoIngestionProperties(_databaseName, _tableName) + { + Format = DataSourceFormat.singlejson, + IngestionMapping = new IngestionMapping() + { + IngestionMappingKind = IngestionMappingKind.Json, + IngestionMappings = McpToolEvent.GetColumnMappings() + } + }; + + var result = await _ingestClient.IngestFromStreamAsync(stream, ingestionProperties); + + if (result != null) + { + _logger.LogInformation("Ingestion results."); + foreach (var item in result.GetIngestionStatusCollection()) + { + _logger.LogInformation("Id: {IngestionSourceId}\tTable: {Table}\tStatus: {Status}\tDetails: {Details}", + item.IngestionSourceId, + item.Table, + item.Status, + item.Details); + } + } + else + { + _logger.LogWarning("Ingestion client did not produce any results."); + } + } + + internal async IAsyncEnumerable GetLatestToolEventsAsync(string kqlFilePath, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (!File.Exists(kqlFilePath)) + { + throw new FileNotFoundException($"KQL file not found: {kqlFilePath}"); + } + + var kql = await File.ReadAllTextAsync(kqlFilePath, cancellationToken); + + var clientRequestProperties = new ClientRequestProperties(); + var reader = await _kustoClient.ExecuteQueryAsync(_databaseName, kql, clientRequestProperties, cancellationToken); + + var eventTimeOrdinal = reader.GetOrdinal("EventTime"); + var eventTypeOrdinal = reader.GetOrdinal("EventType"); + var serverVersionOrdinal = reader.GetOrdinal("ServerVersion"); + var toolIdOrdinal = reader.GetOrdinal("ToolId"); + var toolNameOrdinal = reader.GetOrdinal("ToolName"); + var toolAreaOrdinal = reader.GetOrdinal("ToolArea"); + var replacedByToolNameOrdinal = reader.GetOrdinal("ReplacedByToolName"); + var replacedByToolAreaOrdinal = reader.GetOrdinal("ReplacedByToolArea"); + + while (reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + + var eventTime = reader.GetDateTime(eventTimeOrdinal); + var eventTypeString = reader.GetString(eventTypeOrdinal); + var serverVersion = reader.GetString(serverVersionOrdinal); + var toolId = reader.GetString(toolIdOrdinal); + var toolName = reader.GetString(toolNameOrdinal); + var toolArea = reader.GetString(toolAreaOrdinal); + var replacedByToolName = reader.IsDBNull(replacedByToolNameOrdinal) + ? null + : reader.GetString(replacedByToolNameOrdinal); + var replacedByToolArea = reader.IsDBNull(replacedByToolAreaOrdinal) + ? null + : reader.GetString(replacedByToolAreaOrdinal); + + if (!Enum.TryParse(eventTypeString, ignoreCase: true, out var eventType)) + { + throw new InvalidOperationException($"Invalid EventType value: '{eventTypeString}'. EventTime: '{eventTime}', ToolName: '{toolName}', ToolArea: '{toolArea}'"); + } + + var tool = new McpToolEvent + { + EventTime = eventTime, + EventType = eventType, + ServerVersion = serverVersion, + ToolId = toolId, + ToolName = toolName, + ToolArea = toolArea, + ReplacedByToolName = replacedByToolName, + ReplacedByToolArea = replacedByToolArea, + }; + + yield return tool; + } + } +} diff --git a/eng/tools/ToolMetadataExporter/src/Services/IAzureMcpDatastore.cs b/eng/tools/ToolMetadataExporter/src/Services/IAzureMcpDatastore.cs new file mode 100644 index 0000000000..6a20206c51 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Services/IAzureMcpDatastore.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; + +namespace ToolMetadataExporter.Services; + +public interface IAzureMcpDatastore +{ + Task> GetAvailableToolsAsync(CancellationToken cancellationToken = default); + + Task AddToolEventsAsync(List toolEvents, CancellationToken cancellationToken = default); +} diff --git a/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs b/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs new file mode 100644 index 0000000000..828a8cff2f --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/ToolAnalyzer.cs @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; +using ToolMetadataExporter.Services; +using ToolSelection.Models; + +namespace ToolMetadataExporter; + +public class ToolAnalyzer +{ + private const string Separator = "_"; + + private readonly AzmcpProgram _azmcpExe; + private readonly IAzureMcpDatastore _azureMcpDatastore; + private readonly ILogger _logger; + private readonly string _workingDirectory; + private readonly bool _isDryRun; + + public ToolAnalyzer(AzmcpProgram program, IAzureMcpDatastore azureMcpDatastore, + IOptions configuration, ILogger logger) + { + _azmcpExe = program; + _azureMcpDatastore = azureMcpDatastore; + _logger = logger; + + _workingDirectory = configuration.Value.WorkDirectory ?? throw new ArgumentNullException(nameof(AppConfiguration.WorkDirectory)); + ; + _isDryRun = configuration.Value.IsDryRun; + } + + public async Task RunAsync(DateTimeOffset analysisTime, CancellationToken cancellationToken = default) + { + _logger.LogInformation("Starting analysis. IsDryRun: {IsDryRun}", _isDryRun); + + var serverName = await _azmcpExe.GetServerNameAsync(); + var serverVersion = await _azmcpExe.GetServerVersionAsync(); + var currentTools = await _azmcpExe.LoadToolsDynamicallyAsync(); + + if (currentTools == null) + { + _logger.LogError("LoadToolsDynamicallyAsync did not return a result."); + return; + } + else if (currentTools.Tools == null || currentTools.Tools.Count == 0) + { + _logger.LogWarning("azmcp program did not return any tools."); + return; + } + + var existingTools = (await _azureMcpDatastore.GetAvailableToolsAsync(cancellationToken)).ToDictionary(x => x.ToolId); + + if (cancellationToken.IsCancellationRequested) + { + _logger.LogInformation("Analysis was cancelled."); + return; + } + + // Iterate through all the current tools and match them against the + // state Kusto knows about. + // For each tool, if there is no matching Tool, it is a new tool. + // Else, check the ToolName and ToolArea. If either of those are different + // then there is an update. + var changes = new List(); + + foreach (var tool in currentTools.Tools) + { + if (string.IsNullOrEmpty(tool.Id)) + { + throw new InvalidOperationException($"Tool without an id. Name: {tool.Name}. Command: {tool.Command}"); + } + + var toolArea = GetToolArea(tool)?.ToLowerInvariant(); + if (string.IsNullOrEmpty(toolArea)) + { + throw new InvalidOperationException($"Tool without a tool area. Name: {tool.Name}. Command: {tool.Command} Id: {tool.Id}"); + } + + var changeEvent = new McpToolEvent + { + EventTime = analysisTime, + ToolId = tool.Id, + ServerName = serverName, + ServerVersion = serverVersion, + }; + + var commandWithSeparator = tool.Command?.Replace(" ", Separator).ToLowerInvariant(); + + var hasChange = false; + if (existingTools.Remove(tool.Id, out var knownValue)) + { + if (!string.Equals(commandWithSeparator, knownValue.ToolName, StringComparison.OrdinalIgnoreCase) + || !string.Equals(toolArea, knownValue.ToolArea, StringComparison.OrdinalIgnoreCase)) + { + hasChange = true; + + changeEvent.EventType = McpToolEventType.Updated; + changeEvent.ToolName = knownValue.ToolName; + changeEvent.ToolArea = knownValue.ToolArea; + changeEvent.ReplacedByToolName = commandWithSeparator; + changeEvent.ReplacedByToolArea = toolArea; + } + } + else + { + hasChange = true; + changeEvent.EventType = McpToolEventType.Created; + changeEvent.ToolName = commandWithSeparator; + changeEvent.ToolArea = toolArea; + } + + if (hasChange) + { + changes.Add(changeEvent); + } + } + + // We're done iterating through the newest available tools. + // Any remaining entries in `existingTool` are ones that got deleted. + var removals = existingTools.Select(x => new McpToolEvent + { + ServerName = serverName, + ServerVersion = serverVersion, + EventTime = analysisTime, + EventType = McpToolEventType.Deleted, + ToolId = x.Key, + ToolName = x.Value.ToolName, + ToolArea = x.Value.ToolArea, + ReplacedByToolName = null, + ReplacedByToolArea = null, + }); + + changes.AddRange(removals); + + cancellationToken.ThrowIfCancellationRequested(); + + if (!changes.Any()) + { + _logger.LogInformation("No changes made."); + return; + } + + var filename = $"tool_changes.{analysisTime.Ticks}.json"; + var outputFile = Path.Combine(_workingDirectory, filename); + + _logger.LogInformation("Tool updates. Writing output to: {FileName}", outputFile); + + var writerOptions = new JsonWriterOptions + { + Indented = true, + }; + + using (var ms = new MemoryStream()) + using (var jsonWriter = new Utf8JsonWriter(ms, writerOptions)) + { + JsonSerializer.Serialize(jsonWriter, changes, ModelsSerializationContext.Default.ListMcpToolEvent); + + try + { + await File.WriteAllBytesAsync(outputFile, ms.ToArray(), cancellationToken); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error writing to {FileName}", outputFile); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (!_isDryRun) + { + _logger.LogInformation("Updating datastore."); + await _azureMcpDatastore.AddToolEventsAsync(changes, cancellationToken); + } + } + + private static string? GetToolArea(Tool tool) + { + var split = tool.Command?.Split(" ", 2); + return split == null ? null : split[0]; + } +} diff --git a/eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj b/eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj new file mode 100644 index 0000000000..1b302ecd87 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/ToolMetadataExporter.csproj @@ -0,0 +1,39 @@ + + + + Exe + net9.0 + enable + enable + true + + true + + + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + diff --git a/eng/tools/ToolMetadataExporter/src/Utility.cs b/eng/tools/ToolMetadataExporter/src/Utility.cs new file mode 100644 index 0000000000..629118a27e --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/Utility.cs @@ -0,0 +1,311 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using System.Text.RegularExpressions; +using ToolSelection.Models; + +namespace ToolMetadataExporter; + +public class Utility +{ + internal virtual async Task LoadToolsDynamicallyAsync(string serverFile, string workDirectory, bool isCiMode = false) + { + try + { + var output = await ExecuteAzmcpAsync(serverFile, "tools list", isCiMode); + var jsonOutput = GetJsonFromOutput(output); + + if (jsonOutput == null) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw new InvalidOperationException("No JSON output found from azmcp command."); + } + + // Parse the JSON output + var result = JsonSerializer.Deserialize(jsonOutput, SourceGenerationContext.Default.ListToolsResult); + + // Save the dynamically loaded tools to tools.json for future use + if (result != null) + { + await SaveToolsToJsonAsync(result, Path.Combine(workDirectory, "tools.json")); + + Console.WriteLine($"💾 Saved {result.Tools?.Count} tools to tools.json"); + } + + return result; + } + catch (Exception) + { + if (isCiMode) + { + return null; // Graceful fallback in CI + } + + throw; + } + } + + internal async Task GetServerName(string serverFile) + { + var output = await ExecuteAzmcpAsync(serverFile, "--help", checkErrorCode: false); + + string[] array = Regex.Split(output, "\n\r"); + for (int i = 0; i < array.Length; i++) + { + string? line = array[i]; + if (line.StartsWith("Description:")) + { + return array[i + 1].Trim(); + } + } + + throw new InvalidOperationException("Could not find server name"); + } + + internal async Task GetVersionAsync(string serverFile) + { + var output = await ExecuteAzmcpAsync(serverFile, "--version", checkErrorCode: false); + return output.Trim(); + } + + internal string FindAzmcpAsync(string repositoryRoot, bool isCiMode = false) + { + var searchRoots = new List + { + Path.Combine(repositoryRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Debug"), + Path.Combine(repositoryRoot, "servers", "Azure.Mcp.Server", "src", "bin", "Release") + }; + + var candidateNames = new[] { "azmcp.exe", "azmcp", "azmcp.dll" }; + FileInfo? cliArtifact = null; + + foreach (var root in searchRoots.Where(Directory.Exists)) + { + foreach (var name in candidateNames) + { + var found = new DirectoryInfo(root) + .EnumerateFiles(name, SearchOption.AllDirectories) + .FirstOrDefault(); + if (found != null) + { + cliArtifact = found; + break; + } + } + + if (cliArtifact != null) + { + break; + } + } + + if (cliArtifact != null) + { + return cliArtifact.FullName; + } + + if (isCiMode) + { + return string.Empty; // Graceful fallback in CI + } + + throw new FileNotFoundException("Could not locate azmcp CLI artifact in Debug/Release outputs under servers."); + } + + /// + /// Invokes the azmcp executable with the specified arguments and returns the standard output. + /// + /// Assembly to invoke. + /// Arguments to program. + /// True if it is in CI mode. + /// True to check error code and throw an InvalidOperationException if code is not 0. + /// Standard output as a string. + /// If is true and exit code is not 0. + internal virtual async Task ExecuteAzmcpAsync(string serverFile, string arguments, + bool isCiMode = false, bool checkErrorCode = true) + { + var fileInfo = new FileInfo(serverFile); + var isDll = string.Equals(fileInfo.Extension, ".dll", StringComparison.OrdinalIgnoreCase); + var fileName = isDll ? "dotnet" : fileInfo.FullName; + var argumentsToUse = isDll ? $"{fileInfo.FullName} " : arguments; + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = argumentsToUse, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + } + }; + + process.Start(); + + var output = await process.StandardOutput.ReadToEndAsync(); + var error = await process.StandardError.ReadToEndAsync(); + + await process.WaitForExitAsync(); + + if (checkErrorCode && process.ExitCode != 0) + { + if (isCiMode) + { + return string.Empty; // Graceful fallback in CI + } + + throw new InvalidOperationException($"Failed to execute operation '{arguments}' from azmcp: {error}"); + } + + return output; + } + + private static async Task LoadToolsFromJsonAsync(string filePath, bool isCiMode = false) + { + if (!File.Exists(filePath)) + { + if (isCiMode) + { + return null; // Let caller handle this gracefully + } + + throw new FileNotFoundException($"Tools file not found: {filePath}"); + } + + var json = await File.ReadAllTextAsync(filePath); + + // Process the JSON + if (json.StartsWith('\'') && json.EndsWith('\'')) + { + json = json[1..^1]; // Remove first and last characters (quotes) + json = json.Replace("\\'", "'"); // Convert \' --> ' + json = json.Replace("\\\\\"", "'"); // Convert \\" --> ' + } + + var result = JsonSerializer.Deserialize(json, SourceGenerationContext.Default.ListToolsResult); + + return result; + } + + private static string? GetJsonFromOutput(string? output) + { + if (output == null) + { + return null; + } + + // Filter out non-JSON lines (like launch settings messages) + var lines = output.Split('\n'); + var jsonStartIndex = -1; + + for (int i = 0; i < lines.Length; i++) + { + if (lines[i].Trim().StartsWith("{")) + { + jsonStartIndex = i; + + break; + } + } + + if (jsonStartIndex == -1) + { + return null; + } + + return string.Join('\n', lines.Skip(jsonStartIndex)); + } + + private static async Task SaveToolsToJsonAsync(ListToolsResult toolsResult, string filePath) + { + try + { + // Normalize only tool and option descriptions instead of escaping the entire JSON document + if (toolsResult.Tools != null) + { + foreach (var tool in toolsResult.Tools) + { + if (!string.IsNullOrEmpty(tool.Description)) + { + tool.Description = EscapeCharacters(tool.Description); + } + + if (tool.Options != null) + { + foreach (var opt in tool.Options) + { + if (!string.IsNullOrEmpty(opt.Description)) + { + opt.Description = EscapeCharacters(opt.Description); + } + } + } + } + } + + var writerOptions = new JsonWriterOptions + { + Indented = true, + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + using var stream = new MemoryStream(); + using (var jsonWriter = new Utf8JsonWriter(stream, writerOptions)) + { + JsonSerializer.Serialize(jsonWriter, toolsResult, SourceGenerationContext.Default.ListToolsResult); + } + + await File.WriteAllBytesAsync(filePath, stream.ToArray()); + } + catch (Exception ex) + { + Console.WriteLine($"⚠️ Warning: Failed to save tools to {filePath}: {ex.Message}"); + Console.WriteLine(ex.StackTrace); + } + } + + private static string EscapeCharacters(string text) + { + if (string.IsNullOrEmpty(text)) + return text; + + // Normalize only the fancy “curly” quotes to straight ASCII. Identity replacements were removed. + return text.Replace(UnicodeChars.LeftSingleQuote, "'") + .Replace(UnicodeChars.RightSingleQuote, "'") + .Replace(UnicodeChars.LeftDoubleQuote, "\"") + .Replace(UnicodeChars.RightDoubleQuote, "\""); + } + + // Traverse up from a starting directory to find the repo root (containing AzureMcp.sln or .git) + internal static string FindRepoRoot(string startDir) + { + var dir = new DirectoryInfo(startDir); + + while (dir != null) + { + if (File.Exists(Path.Combine(dir.FullName, "AzureMcp.sln")) || + Directory.Exists(Path.Combine(dir.FullName, ".git"))) + { + return dir.FullName; + } + dir = dir.Parent; + } + + throw new InvalidOperationException("Could not find repo root (AzureMcp.sln or .git)."); + } + + internal static class UnicodeChars + { + public const string LeftSingleQuote = "\u2018"; + public const string RightSingleQuote = "\u2019"; + public const string LeftDoubleQuote = "\u201C"; + public const string RightDoubleQuote = "\u201D"; + } +} diff --git a/eng/tools/ToolMetadataExporter/src/appsettings.Development.json b/eng/tools/ToolMetadataExporter/src/appsettings.Development.json new file mode 100644 index 0000000000..0203ba7a47 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/appsettings.Development.json @@ -0,0 +1,12 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Debug" + } + }, + "AppConfig": { + "IngestionEndpoint": "https://ingest-test.westus.kusto.windows.net", + "QueryEndpoint": "https://test.westus.kusto.windows.net", + "DatabaseName": "McpToolTest" + } +} diff --git a/eng/tools/ToolMetadataExporter/src/appsettings.json b/eng/tools/ToolMetadataExporter/src/appsettings.json new file mode 100644 index 0000000000..fde5467e3f --- /dev/null +++ b/eng/tools/ToolMetadataExporter/src/appsettings.json @@ -0,0 +1,8 @@ +{ + "AppConfig": { + "IngestionEndpoint": "https://ingest-your-server.region.kusto.windows.net", + "QueryEndpoint": "https://your-server.region.kusto.windows.net", + "DatabaseName": "McpDatastore", + "McpToolEventsTableName": "McpToolEvents" + } +} diff --git a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzmcpProgramTests.cs b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzmcpProgramTests.cs new file mode 100644 index 0000000000..3d791ff44d --- /dev/null +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzmcpProgramTests.cs @@ -0,0 +1,379 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Services; +using Xunit; + +namespace ToolMetadataExporter.UnitTests.Services; + +public class AzmcpProgramTests +{ + private readonly Utility _utility; + private readonly ILogger _logger; + private readonly IOptions _options; + private readonly AppConfiguration _appConfiguration; + + public AzmcpProgramTests() + { + _utility = Substitute.For(); + _logger = Substitute.For>(); + _options = Substitute.For>(); + + _appConfiguration = new AppConfiguration + { + AzmcpExe = "azmcp", + WorkDirectory = "/tmp" + }; + + _options.Value.Returns(_appConfiguration); + } + + /// + /// Verifies that the server name is correctly retrieved from the server information response and is lowercase. + /// + [Fact] + public async Task GetsServerNameWithServerInfo() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Template.Mcp.Server", Version = "1.0.0-beta.1+20-0-2" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerNameAsync(); + + Assert.Equal(serverInfo.Name.ToLowerInvariant(), actual); + } + + /// + /// Verifies that the server name is correctly retrieved from the help command output when server info fails. + /// Checks that the name is formatted to lowercase with dots replacing spaces. + /// + [Fact] + public async Task GetsServerNameWithHelp() + { + // Arrange + var serverInfoOutput = + """ + Required command was not provided. + Unrecognized command or argument 'info'. + + Description: + MCP Server operations - Commands for managing and interacting with the MCP Server. + + Usage: + azmcp server [command] [options] + + Options: + -?, -h, --help Show help and usage information + """; + var helpOutput = """ + Description: + Template MCP Server + + Usage: + azmcp [command] [options] + + Options: + -?, -h, --help Show help and usage information + --version Show version information + + Commands: + get_bestpractices Azure best practices - Commands return a list of best practices for code generation, + """; + + _utility.ExecuteAzmcpAsync(Arg.Any(), "server info", checkErrorCode: false).Returns(serverInfoOutput); + _utility.ExecuteAzmcpAsync(Arg.Any(), "--help", checkErrorCode: false).Returns(helpOutput); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerNameAsync(); + + Assert.Equal("template.mcp.server", actual); + } + + [Fact] + public async Task GetsServerVersionFromServerInfo() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Azure.Mcp.Server", Version = "1.2.3-beta.4" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("1.2.3-beta.4", actual); + } + + [Fact] + public async Task GetsServerVersionFromVersionCommand() + { + // Arrange + var serverInfoOutput = "invalid json"; + var versionOutput = "2.0.1"; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serverInfoOutput)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--version", checkErrorCode: false).Returns(Task.FromResult(versionOutput)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("2.0.1", actual); + } + + [Fact] + public async Task GetsServerVersionStripsGitHash() + { + // Arrange + var serverInfoOutput = "invalid json"; + var versionOutput = "1.5.0+abc123def456"; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serverInfoOutput)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--version", checkErrorCode: false).Returns(Task.FromResult(versionOutput)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("1.5.0", actual); + } + + [Fact] + public async Task LoadToolsDynamicallyAsync_CallsUtilityMethod() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Test.Server", Version = "1.0.0" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.LoadToolsDynamicallyAsync(); + + // Assert - Verify the utility method was called + await _utility.Received(1).LoadToolsDynamicallyAsync(_appConfiguration.AzmcpExe!, _appConfiguration.WorkDirectory!, false); + } + + [Fact] + public void Constructor_ThrowsWhenWorkDirectoryIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + AzmcpExe = "azmcp", + WorkDirectory = null + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => new AzmcpProgram(_utility, invalidOptions, _logger)); + } + + [Fact] + public void Constructor_ThrowsWhenAzmcpExeIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + AzmcpExe = null, + WorkDirectory = "/tmp" + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => new AzmcpProgram(_utility, invalidOptions, _logger)); + } + + [Fact] + public async Task GetServerNameAsync_ThrowsWhenBothServerInfoAndHelpFail() + { + // Arrange + var serverInfoOutput = "invalid json"; + var helpOutput = """ + Usage: + azmcp [command] [options] + + Options: + -?, -h, --help Show help and usage information + """; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serverInfoOutput)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--help", checkErrorCode: false).Returns(Task.FromResult(helpOutput)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => await program.GetServerNameAsync()); + Assert.Contains("Failed to determine server name", exception.Message); + } + + [Fact] + public async Task GetServerInfoInternal_ReturnsNullForNullResults() + { + // Arrange + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = null + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + var helpOutput = """ + Description: + Null Results Server + + Usage: + azmcp [command] [options] + """; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--help", checkErrorCode: false).Returns(Task.FromResult(helpOutput)); + + // Act + var program = new AzmcpProgram(_utility, _options, _logger); + var actual = await program.GetServerNameAsync(); + + // Assert + Assert.Equal("null.results.server", actual); + } + + [Fact] + public async Task GetServerVersionAsync_HandlesVersionWithWhitespace() + { + // Arrange + var serverInfoOutput = "invalid json"; + var versionOutput = " 3.2.1 \n"; + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serverInfoOutput)); + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "--version", checkErrorCode: false).Returns(Task.FromResult(versionOutput)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act + var actual = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("3.2.1", actual); + } + + [Fact] + public async Task GetServerNameAsync_CachesResult() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Cached.Server", Version = "1.0.0" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act - Call twice + var firstResult = await program.GetServerNameAsync(); + var secondResult = await program.GetServerNameAsync(); + + // Assert + Assert.Equal("cached.server", firstResult); + Assert.Equal(firstResult, secondResult); + + // Verify ExecuteAzmcpAsync was only called once for server info (during construction) + await _utility.Received(1).ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false); + } + + [Fact] + public async Task GetServerVersionAsync_CachesResult() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Test.Server", Version = "5.6.7" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act - Call twice + var firstResult = await program.GetServerVersionAsync(); + var secondResult = await program.GetServerVersionAsync(); + + // Assert + Assert.Equal("5.6.7", firstResult); + Assert.Equal(firstResult, secondResult); + + // Verify ExecuteAzmcpAsync was only called once for server info (during construction) + await _utility.Received(1).ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false); + } + + [Fact] + public async Task LoadToolsDynamicallyAsync_UsesLazyInitialization() + { + // Arrange + var serverInfo = new ServerInfo { Name = "Test.Server", Version = "1.0.0" }; + var serverInfoResult = new ServerInfoResult + { + Status = 200, + Message = "Success", + Results = serverInfo + }; + var serialized = JsonSerializer.Serialize(serverInfoResult, ModelsSerializationContext.Default.ServerInfoResult); + + _utility.ExecuteAzmcpAsync(_appConfiguration.AzmcpExe!, "server info", checkErrorCode: false).Returns(Task.FromResult(serialized)); + + var program = new AzmcpProgram(_utility, _options, _logger); + + // Act - Call twice to verify lazy initialization + var firstResult = await program.LoadToolsDynamicallyAsync(); + var secondResult = await program.LoadToolsDynamicallyAsync(); + + // Assert - Verify LoadToolsDynamicallyAsync was only called once (lazy initialization) + await _utility.Received(1).LoadToolsDynamicallyAsync(_appConfiguration.AzmcpExe!, _appConfiguration.WorkDirectory!, false); + } +} diff --git a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs new file mode 100644 index 0000000000..a7a1db618c --- /dev/null +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/Services/AzureMcpKustoDatastoreTests.cs @@ -0,0 +1,689 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Data; +using Kusto.Data.Common; +using Kusto.Ingest; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using ToolMetadataExporter.Models.Kusto; +using ToolMetadataExporter.Services; +using Xunit; + +namespace ToolMetadataExporter.UnitTests.Services; + +public class AzureMcpKustoDatastoreTests : IDisposable +{ + private readonly ICslQueryProvider _kustoClient; + private readonly IKustoIngestClient _ingestClient; + private readonly ILogger _logger; + private readonly IOptions _options; + private readonly AppConfiguration _appConfiguration; + private readonly string _tempQueriesDirectory; + + public AzureMcpKustoDatastoreTests() + { + _kustoClient = Substitute.For(); + _ingestClient = Substitute.For(); + _logger = Substitute.For>(); + _options = Substitute.For>(); + + // Create a temporary directory for queries + _tempQueriesDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(_tempQueriesDirectory); + + _appConfiguration = new AppConfiguration + { + DatabaseName = "TestDatabase", + McpToolEventsTableName = "McpToolEvents", + QueriesFolder = _tempQueriesDirectory + }; + + _options.Value.Returns(_appConfiguration); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_tempQueriesDirectory)) + { + Directory.Delete(_tempQueriesDirectory, recursive: true); + } + } + catch (Exception) + { + // Suppress cleanup exceptions to avoid failing tests + // The OS will eventually clean up temp directories + } + } + + [Fact] + public void Constructor_ThrowsWhenDatabaseNameIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + DatabaseName = null, + McpToolEventsTableName = "McpToolEvents", + QueriesFolder = _tempQueriesDirectory + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => + new AzureMcpKustoDatastore(_kustoClient, _ingestClient, invalidOptions, _logger)); + } + + [Fact] + public void Constructor_ThrowsWhenTableNameIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + DatabaseName = "TestDatabase", + McpToolEventsTableName = null, + QueriesFolder = _tempQueriesDirectory + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => + new AzureMcpKustoDatastore(_kustoClient, _ingestClient, invalidOptions, _logger)); + } + + [Fact] + public void Constructor_ThrowsWhenQueriesFolderIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + DatabaseName = "TestDatabase", + McpToolEventsTableName = "McpToolEvents", + QueriesFolder = null + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => + new AzureMcpKustoDatastore(_kustoClient, _ingestClient, invalidOptions, _logger)); + } + + [Fact] + public void Constructor_ThrowsWhenQueriesFolderDoesNotExist() + { + // Arrange + var nonExistentPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var invalidConfig = new AppConfiguration + { + DatabaseName = "TestDatabase", + McpToolEventsTableName = "McpToolEvents", + QueriesFolder = nonExistentPath + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + var exception = Assert.Throws(() => + new AzureMcpKustoDatastore(_kustoClient, _ingestClient, invalidOptions, _logger)); + Assert.Contains("does not exist", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsWhenQueryFileNotFound() + { + // Arrange + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("Could not find GetAvailableTools.kql", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ReturnsCreatedTools() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + var result = await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.Single(result); + Assert.Equal("tool-1", result[0].ToolId); + Assert.Equal("TestTool", result[0].ToolName); + Assert.Equal("TestArea", result[0].ToolArea); + } + + [Fact] + public async Task GetAvailableToolsAsync_ReturnsUpdatedTools() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Updated, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "OldTool", + ToolArea = "OldArea", + ReplacedByToolName = "NewTool", + ReplacedByToolArea = "NewArea" + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + var result = await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.Single(result); + Assert.Equal("tool-1", result[0].ToolId); + Assert.Equal("NewTool", result[0].ToolName); + Assert.Equal("NewArea", result[0].ToolArea); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsForUnsupportedEventType() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Deleted, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("unsupported event type", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsWhenToolIdIsEmpty() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("Cannot have an event with no id", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsWhenToolNameIsEmpty() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("without tool name and/or a tool area", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ThrowsWhenToolAreaIsEmpty() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken)); + Assert.Contains("without tool name and/or a tool area", exception.Message); + } + + [Fact] + public async Task GetAvailableToolsAsync_ProcessesMultipleTools() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "GetAvailableTools.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "Tool1", + ToolArea = "Area1", + ReplacedByToolName = null, + ReplacedByToolArea = null + }, + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Updated, + ServerVersion = "1.0.0", + ToolId = "tool-2", + ToolName = "OldTool2", + ToolArea = "OldArea2", + ReplacedByToolName = "NewTool2", + ReplacedByToolArea = "NewArea2" + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + var result = await datastore.GetAvailableToolsAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.Equal(2, result.Count); + Assert.Equal("tool-1", result[0].ToolId); + Assert.Equal("Tool1", result[0].ToolName); + Assert.Equal("tool-2", result[1].ToolId); + Assert.Equal("NewTool2", result[1].ToolName); + } + + [Fact] + public async Task AddToolEventsAsync_IngestsDataSuccessfully() + { + // Arrange + var toolEvents = new List + { + new() + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }; + + var mockIngestionStatus = Substitute.For(); + var statusCollection = new List + { + new() + { + IngestionSourceId = Guid.NewGuid(), + Table = "McpToolEvents", + Status = Status.Succeeded, + Details = "Success" + } + }; + mockIngestionStatus.GetIngestionStatusCollection().Returns(statusCollection); + + _ingestClient.IngestFromStreamAsync( + Arg.Any(), + Arg.Any()) + .Returns(mockIngestionStatus); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + await datastore.AddToolEventsAsync(toolEvents, TestContext.Current.CancellationToken); + + // Assert + await _ingestClient.Received(1).IngestFromStreamAsync( + Arg.Any(), + Arg.Is(p => + p.DatabaseName == "TestDatabase" && + p.TableName == "McpToolEvents")); + } + + [Fact] + public async Task AddToolEventsAsync_LogsWarningWhenResultIsNull() + { + // Arrange + var toolEvents = new List + { + new() + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }; + + _ingestClient.IngestFromStreamAsync( + Arg.Any(), + Arg.Any()) + .Returns((IKustoIngestionResult?)null); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + await datastore.AddToolEventsAsync(toolEvents, TestContext.Current.CancellationToken); + + // Assert + await _ingestClient.Received(1).IngestFromStreamAsync( + Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task AddToolEventsAsync_HandlesEmptyList() + { + // Arrange + var toolEvents = new List(); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + await datastore.AddToolEventsAsync(toolEvents, TestContext.Current.CancellationToken); + + // Assert + await _ingestClient.Received(1).IngestFromStreamAsync( + Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task GetLatestToolEventsAsync_ThrowsWhenKqlFileNotFound() + { + // Arrange + var nonExistentFile = Path.Combine(_tempQueriesDirectory, "NonExistent.kql"); + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in datastore.GetLatestToolEventsAsync(nonExistentFile, TestContext.Current.CancellationToken)) + { + // Should not reach here + } + }); + } + + [Fact] + public async Task GetLatestToolEventsAsync_ReadsEventsFromKusto() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "TestQuery.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = CreateMockDataReader(new[] + { + new McpToolEvent + { + EventTime = DateTime.UtcNow, + EventType = McpToolEventType.Created, + ServerVersion = "1.0.0", + ToolId = "tool-1", + ToolName = "TestTool", + ToolArea = "TestArea", + ReplacedByToolName = null, + ReplacedByToolArea = null + } + }); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act + var results = new List(); + await foreach (var toolEvent in datastore.GetLatestToolEventsAsync(queryFile, TestContext.Current.CancellationToken)) + { + results.Add(toolEvent); + } + + // Assert + Assert.Single(results); + Assert.Equal("tool-1", results[0].ToolId); + Assert.Equal("TestTool", results[0].ToolName); + } + + [Fact] + public async Task GetLatestToolEventsAsync_ThrowsOnInvalidEventType() + { + // Arrange + var queryFile = Path.Combine(_tempQueriesDirectory, "TestQuery.kql"); + await File.WriteAllTextAsync(queryFile, "test query", TestContext.Current.CancellationToken); + + var mockReader = Substitute.For(); + mockReader.Read().Returns(true, false); + mockReader.GetOrdinal("EventTime").Returns(0); + mockReader.GetOrdinal("EventType").Returns(1); + mockReader.GetOrdinal("ServerVersion").Returns(2); + mockReader.GetOrdinal("ToolId").Returns(3); + mockReader.GetOrdinal("ToolName").Returns(4); + mockReader.GetOrdinal("ToolArea").Returns(5); + mockReader.GetOrdinal("ReplacedByToolName").Returns(6); + mockReader.GetOrdinal("ReplacedByToolArea").Returns(7); + + mockReader.GetDateTime(0).Returns(DateTime.UtcNow); + mockReader.GetString(1).Returns("InvalidEventType"); + mockReader.GetString(2).Returns("1.0.0"); + mockReader.GetString(3).Returns("tool-1"); + mockReader.GetString(4).Returns("TestTool"); + mockReader.GetString(5).Returns("TestArea"); + mockReader.IsDBNull(6).Returns(true); + mockReader.IsDBNull(7).Returns(true); + + _kustoClient.ExecuteQueryAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(mockReader); + + var datastore = new AzureMcpKustoDatastore(_kustoClient, _ingestClient, _options, _logger); + + // Act & Assert + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in datastore.GetLatestToolEventsAsync(queryFile, TestContext.Current.CancellationToken)) + { + // Should not reach here + } + }); + } + + private static IDataReader CreateMockDataReader(McpToolEvent[] events) + { + var reader = Substitute.For(); + var currentIndex = -1; + + reader.Read().Returns(callInfo => + { + currentIndex++; + return currentIndex < events.Length; + }); + + reader.GetOrdinal("EventTime").Returns(0); + reader.GetOrdinal("EventType").Returns(1); + reader.GetOrdinal("ServerVersion").Returns(2); + reader.GetOrdinal("ToolId").Returns(3); + reader.GetOrdinal("ToolName").Returns(4); + reader.GetOrdinal("ToolArea").Returns(5); + reader.GetOrdinal("ReplacedByToolName").Returns(6); + reader.GetOrdinal("ReplacedByToolArea").Returns(7); + + reader.GetDateTime(Arg.Is(0)).Returns(x => GetCurrentEventTime()); + reader.GetString(Arg.Is(1)).Returns(x => GetCurrentEventType()); + reader.GetString(Arg.Is(2)).Returns(x => GetCurrentServerVersion()); + reader.GetString(Arg.Is(3)).Returns(x => GetCurrentToolId()); + reader.GetString(Arg.Is(4)).Returns(x => GetCurrentToolName()); + reader.GetString(Arg.Is(5)).Returns(x => GetCurrentToolArea()); + + reader.IsDBNull(Arg.Is(6)).Returns(x => IsReplacedByToolNameNull()); + reader.GetString(Arg.Is(6)).Returns(x => GetCurrentReplacedByToolName()); + + reader.IsDBNull(Arg.Is(7)).Returns(x => IsReplacedByToolAreaNull()); + reader.GetString(Arg.Is(7)).Returns(x => GetCurrentReplacedByToolArea()); + + return reader; + + DateTime GetCurrentEventTime() => events[Math.Max(0, currentIndex)].EventTime?.DateTime ?? DateTime.UtcNow; + string GetCurrentEventType() => events[Math.Max(0, currentIndex)].EventType?.ToString() ?? string.Empty; + string GetCurrentServerVersion() => events[Math.Max(0, currentIndex)].ServerVersion ?? string.Empty; + string GetCurrentToolId() => events[Math.Max(0, currentIndex)].ToolId ?? string.Empty; + string GetCurrentToolName() => events[Math.Max(0, currentIndex)].ToolName ?? string.Empty; + string GetCurrentToolArea() => events[Math.Max(0, currentIndex)].ToolArea ?? string.Empty; + bool IsReplacedByToolNameNull() => string.IsNullOrEmpty(events[Math.Max(0, currentIndex)].ReplacedByToolName); + string GetCurrentReplacedByToolName() => events[Math.Max(0, currentIndex)].ReplacedByToolName ?? string.Empty; + bool IsReplacedByToolAreaNull() => string.IsNullOrEmpty(events[Math.Max(0, currentIndex)].ReplacedByToolArea); + string GetCurrentReplacedByToolArea() => events[Math.Max(0, currentIndex)].ReplacedByToolArea ?? string.Empty; + } +} diff --git a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs new file mode 100644 index 0000000000..5b54fe4b30 --- /dev/null +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolAnalyzerTests.cs @@ -0,0 +1,575 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using ToolMetadataExporter.Models; +using ToolMetadataExporter.Models.Kusto; +using ToolMetadataExporter.Services; +using ToolSelection.Models; +using Xunit; + +namespace ToolMetadataExporter.UnitTests; + +public class ToolAnalyzerTests : IDisposable +{ + private readonly AzmcpProgram _azmcpProgram; + private readonly IAzureMcpDatastore _datastore; + private readonly ILogger _logger; + private readonly IOptions _options; + private readonly AppConfiguration _appConfiguration; + private readonly string _tempWorkingDirectory; + + public ToolAnalyzerTests() + { + var utility = Substitute.For(); + var logger = Substitute.For>(); + var programOptions = Substitute.For>(); + var programConfig = new AppConfiguration + { + WorkDirectory = Path.GetTempPath(), + AzmcpExe = "azmcp" + }; + programOptions.Value.Returns(programConfig); + + _azmcpProgram = Substitute.ForPartsOf(utility, programOptions, logger); + _datastore = Substitute.For(); + _logger = Substitute.For>(); + _options = Substitute.For>(); + + // Create a temporary directory for working files + _tempWorkingDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(_tempWorkingDirectory); + + _appConfiguration = new AppConfiguration + { + WorkDirectory = _tempWorkingDirectory, + IsDryRun = false + }; + + _options.Value.Returns(_appConfiguration); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_tempWorkingDirectory)) + { + Directory.Delete(_tempWorkingDirectory, recursive: true); + } + } + catch (Exception) + { + // Suppress cleanup exceptions to avoid failing tests + } + } + + [Fact] + public void Constructor_ThrowsWhenWorkDirectoryIsNull() + { + // Arrange + var invalidConfig = new AppConfiguration + { + WorkDirectory = null, + IsDryRun = false + }; + var invalidOptions = Substitute.For>(); + invalidOptions.Value.Returns(invalidConfig); + + // Act & Assert + Assert.Throws(() => + new ToolAnalyzer(_azmcpProgram, _datastore, invalidOptions, _logger)); + } + + [Fact] + public async Task RunAsync_ReturnsEarly_WhenLoadToolsDynamicallyReturnsNull() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(null)); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().GetAvailableToolsAsync(Arg.Any()); + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_ReturnsEarly_WhenToolsListIsNull() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult { Tools = null })); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().GetAvailableToolsAsync(Arg.Any()); + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_ReturnsEarly_WhenToolsListIsEmpty() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult { Tools = [] })); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().GetAvailableToolsAsync(Arg.Any()); + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_ThrowsException_WhenToolHasNoId() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = null, Name = "test-tool", Command = "area command" } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken)); + Assert.Contains("Tool without an id", exception.Message); + } + + [Fact] + public async Task RunAsync_ThrowsException_WhenToolHasNoCommand() + { + // Arrange + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = "tool-1", Name = "test-tool", Command = null } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken)); + Assert.Contains("Tool without a tool area", exception.Message); + } + + [Fact] + public async Task RunAsync_DetectsNewTool() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var serverName = "test-server"; + var serverVersion = "1.0.0"; + var tool = new Tool { Id = "tool-1", Name = "New Tool", Command = "area command" }; + var expectedToolName = "area_command"; + var expectedToolArea = "area"; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult(serverName)); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult(serverVersion)); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].EventType == McpToolEventType.Created && + events[0].ToolId == tool.Id && + events[0].ToolName == expectedToolName && + events[0].ToolArea == expectedToolArea && + events[0].ServerName == serverName && + events[0].ServerVersion == serverVersion), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_DetectsUpdatedTool() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var serverName = "test-server"; + var serverVersion = "1.0.0"; + var tool = new Tool { Id = "tool-1", Name = "Updated Tool", Command = "newarea newcommand" }; + var existingTool = new AzureMcpTool("tool-1", "oldarea_oldcommand", "oldarea"); + var expectedNewToolName = "newarea_newcommand"; + var expectedNewToolArea = "newarea"; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult(serverName)); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult(serverVersion)); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].EventType == McpToolEventType.Updated && + events[0].ToolId == tool.Id && + events[0].ToolName == existingTool.ToolName && + events[0].ToolArea == existingTool.ToolArea && + events[0].ReplacedByToolName == expectedNewToolName && + events[0].ReplacedByToolArea == expectedNewToolArea), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_DetectsDeletedTool() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var serverName = "test-server"; + var serverVersion = "1.0.0"; + var existingTool = new AzureMcpTool("tool-1", "area_command", "area"); + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult(serverName)); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult(serverVersion)); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].EventType == McpToolEventType.Deleted && + events[0].ToolId == existingTool.ToolId && + events[0].ToolName == existingTool.ToolName && + events[0].ToolArea == existingTool.ToolArea && + events[0].ReplacedByToolName == null && + events[0].ReplacedByToolArea == null), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_DoesNotDetectChange_WhenToolUnchanged() + { + // Arrange + var tool = new Tool { Id = "tool-1", Name = "Test Tool", Command = "area command" }; + var existingTool = new AzureMcpTool("tool-1", "area_command", "area"); + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_HandlesMultipleChanges() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var tool1 = new Tool { Id = "tool-1", Name = "Tool 1", Command = "area1 command1" }; // Unchanged + var tool2 = new Tool { Id = "tool-2", Name = "Tool 2", Command = "area2 newcommand" }; // Updated + var tool4 = new Tool { Id = "tool-4", Name = "Tool 4", Command = "area4 command4" }; // New + var existingTool1 = new AzureMcpTool("tool-1", "area1_command1", "area1"); + var existingTool2 = new AzureMcpTool("tool-2", "area2_oldcommand", "area2"); + var existingTool3 = new AzureMcpTool("tool-3", "area3_command3", "area3"); // Deleted + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool1, tool2, tool4] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool1, + existingTool2, + existingTool3 + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 3 && + events.Any(e => e.EventType == McpToolEventType.Updated && e.ToolId == tool2.Id) && + events.Any(e => e.EventType == McpToolEventType.Created && e.ToolId == tool4.Id) && + events.Any(e => e.EventType == McpToolEventType.Deleted && e.ToolId == existingTool3.ToolId)), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_WritesChangesToFile() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var tool = new Tool { Id = "tool-1", Name = "New Tool", Command = "area command" }; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + var outputFile = Path.Combine(_tempWorkingDirectory, "tool_changes.json"); + Assert.True(File.Exists(outputFile)); + var fileContent = await File.ReadAllTextAsync(outputFile, TestContext.Current.CancellationToken); + Assert.Contains(tool.Id, fileContent); + Assert.Contains("Created", fileContent); + } + + [Fact] + public async Task RunAsync_SkipsDatastoreUpdate_WhenDryRunIsTrue() + { + // Arrange + _appConfiguration.IsDryRun = true; + var analysisTime = DateTimeOffset.UtcNow; + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = "tool-1", Name = "New Tool", Command = "area command" } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + + // But file should still be written + var outputFile = Path.Combine(_tempWorkingDirectory, "tool_changes.json"); + Assert.True(File.Exists(outputFile)); + } + + [Fact] + public async Task RunAsync_HandlesCancellation() + { + // Arrange + var cts = new CancellationTokenSource(); + cts.Cancel(); + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = "tool-1", Name = "Test Tool", Command = "area command" } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, cts.Token); + + // Assert - Should return early without throwing + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task RunAsync_NormalizesCommandToLowercase() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var tool = new Tool { Id = "tool-1", Name = "Test Tool", Command = "Area Command" }; + var expectedToolName = "area_command"; + var expectedToolArea = "area"; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].ToolName == expectedToolName && + events[0].ToolArea == expectedToolArea), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_ReplacesSpacesWithUnderscores() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = + [ + new Tool { Id = "tool-1", Name = "Test Tool", Command = "area command with spaces" } + ] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>([])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].ToolName == "area_command_with_spaces"), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_DetectsToolAreaChange() + { + // Arrange + var analysisTime = DateTimeOffset.UtcNow; + var tool = new Tool { Id = "tool-1", Name = "Test Tool", Command = "newarea command" }; + var existingTool = new AzureMcpTool("tool-1", "oldarea_command", "oldarea"); + var expectedNewToolArea = "newarea"; + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(analysisTime, TestContext.Current.CancellationToken); + + // Assert + await _datastore.Received(1).AddToolEventsAsync( + Arg.Is>(events => + events.Count == 1 && + events[0].EventType == McpToolEventType.Updated && + events[0].ToolArea == existingTool.ToolArea && + events[0].ReplacedByToolArea == expectedNewToolArea), + Arg.Any()); + } + + [Fact] + public async Task RunAsync_IsCaseInsensitive_ForComparison() + { + // Arrange + var tool = new Tool { Id = "tool-1", Name = "Test Tool", Command = "AREA COMMAND" }; + var existingTool = new AzureMcpTool("tool-1", "area_command", "area"); + + _azmcpProgram.GetServerNameAsync().Returns(Task.FromResult("test-server")); + _azmcpProgram.GetServerVersionAsync().Returns(Task.FromResult("1.0.0")); + _azmcpProgram.LoadToolsDynamicallyAsync().Returns(Task.FromResult(new ListToolsResult + { + Tools = [tool] + })); + _datastore.GetAvailableToolsAsync(Arg.Any()).Returns(Task.FromResult>( + [ + existingTool + ])); + + var analyzer = new ToolAnalyzer(_azmcpProgram, _datastore, _options, _logger); + + // Act + await analyzer.RunAsync(DateTimeOffset.UtcNow, TestContext.Current.CancellationToken); + + // Assert - Should not detect any changes + await _datastore.DidNotReceive().AddToolEventsAsync(Arg.Any>(), Arg.Any()); + } +} diff --git a/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj new file mode 100644 index 0000000000..49f3f59f9c --- /dev/null +++ b/eng/tools/ToolMetadataExporter/tests/ToolMetadataExporter.UnitTests/ToolMetadataExporter.UnitTests.csproj @@ -0,0 +1,16 @@ + + + true + Exe + + + + + + + + + + + + diff --git a/eng/tools/Tools.sln b/eng/tools/Tools.sln index febcf965b4..5267747a7a 100644 --- a/eng/tools/Tools.sln +++ b/eng/tools/Tools.sln @@ -6,6 +6,18 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ToolDescriptionEvaluator", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolDescriptionEvaluator", "ToolDescriptionEvaluator\src\ToolDescriptionEvaluator.csproj", "{CE346AEC-0DD6-C210-A058-B6D879AF55B1}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ToolMetadataExporter", "ToolMetadataExporter", "{F86DD719-7EB8-4D49-81EA-F2D6565A7047}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{B8D43586-6D54-4A71-A714-D7AC92AE64DD}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{B28D2E97-9CDA-4BF2-95EC-CF169C6339EC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter", "ToolMetadataExporter\src\ToolMetadataExporter.csproj", "{41A27DC4-E2C0-E36D-EF8E-854DFC376331}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToolMetadataExporter.UnitTests", "ToolMetadataExporter\tests\ToolMetadataExporter.UnitTests\ToolMetadataExporter.UnitTests.csproj", "{8B9656E4-84BE-0199-1E72-B4E5B713E059}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -16,14 +28,27 @@ Global {CE346AEC-0DD6-C210-A058-B6D879AF55B1}.Debug|Any CPU.Build.0 = Debug|Any CPU {CE346AEC-0DD6-C210-A058-B6D879AF55B1}.Release|Any CPU.ActiveCfg = Release|Any CPU {CE346AEC-0DD6-C210-A058-B6D879AF55B1}.Release|Any CPU.Build.0 = Release|Any CPU + {41A27DC4-E2C0-E36D-EF8E-854DFC376331}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {41A27DC4-E2C0-E36D-EF8E-854DFC376331}.Debug|Any CPU.Build.0 = Debug|Any CPU + {41A27DC4-E2C0-E36D-EF8E-854DFC376331}.Release|Any CPU.ActiveCfg = Release|Any CPU + {41A27DC4-E2C0-E36D-EF8E-854DFC376331}.Release|Any CPU.Build.0 = Release|Any CPU + {8B9656E4-84BE-0199-1E72-B4E5B713E059}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8B9656E4-84BE-0199-1E72-B4E5B713E059}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8B9656E4-84BE-0199-1E72-B4E5B713E059}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8B9656E4-84BE-0199-1E72-B4E5B713E059}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution - {CE346AEC-0DD6-C210-A058-B6D879AF55B1} = {89C7D52E-46CB-E5E4-EB56-C23691DF38B5} + {CE346AEC-0DD6-C210-A058-B6D879AF55B1} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} + {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} = {89C7D52E-46CB-E5E4-EB56-C23691DF38B5} + {B8D43586-6D54-4A71-A714-D7AC92AE64DD} = {F86DD719-7EB8-4D49-81EA-F2D6565A7047} + {B28D2E97-9CDA-4BF2-95EC-CF169C6339EC} = {F86DD719-7EB8-4D49-81EA-F2D6565A7047} + {41A27DC4-E2C0-E36D-EF8E-854DFC376331} = {B8D43586-6D54-4A71-A714-D7AC92AE64DD} + {8B9656E4-84BE-0199-1E72-B4E5B713E059} = {B28D2E97-9CDA-4BF2-95EC-CF169C6339EC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FE4B2877-CB8E-40CE-92EA-A30A5B9C07EE} EndGlobalSection -EndGlobal \ No newline at end of file +EndGlobal