diff --git a/docs/content/api-overview/resources/loganalytics.md b/docs/content/api-overview/resources/loganalytics.md index 1101625cd..4ea3d53d9 100644 --- a/docs/content/api-overview/resources/loganalytics.md +++ b/docs/content/api-overview/resources/loganalytics.md @@ -10,8 +10,9 @@ weight: 12 The Log Analytics builder is used to create Work space instances. - Log Analytics (`Microsoft.OperationalInsights/workspaces`) +- Tables (`Microsoft.OperationalInsights/workspaces/tables`) -#### Builder Keywords +#### Log Analytics Builder Keywords | Keyword | Purpose | | ---------------- | --------------------------------------------------------------- | @@ -20,9 +21,19 @@ The Log Analytics builder is used to create Work space instances. | enable_ingestion | Enables ingestion network traffic. | | enable_query | Enables query network traffic. | | daily_cap | Specifies an upper limit on the amount of data to ingest daily. | +| tables | Defines tables to be created in the workspace. | | add_tags | Adds a set of tags to the resource | | add_tag | Adds a tag to the resource | +#### Table Builder Keywords + +| Keyword | Purpose | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------| +| Name | Sets the name of the table. | +| Plan | Sets the table plan. Analytics plans can set a retention period between 4 and 730. If ommited will default to the workspace retention.| +| Columns | Sets the columns of the table. Each column has a name and type. | +| TotalRetentionInDays | Sets the total retention period for the table in days, between 4 and 4383. If ommited will default to the Plan retention. | + #### Configuration Members | Member | Purpose | @@ -42,6 +53,17 @@ let myAnalytics = logAnalytics { enable_ingestion enable_query daily_cap 5 + tables [ + { + Name = ResourceName "Serilog" + Plan = Analytics (Some 30) + Columns = [ + { Name = "TimeGenerated"; Type = "datetime" } + { Name = "Event"; Type = "dynamic" } + ] + TotalRetentionInDays = None + } + ] add_tag "tag1" "myTestResourceFarmer" } diff --git a/src/Farmer/Arm/LogAnalytics.fs b/src/Farmer/Arm/LogAnalytics.fs index 95353959e..912dfbe46 100644 --- a/src/Farmer/Arm/LogAnalytics.fs +++ b/src/Farmer/Arm/LogAnalytics.fs @@ -6,6 +6,59 @@ open Farmer let workspaces = ResourceType("Microsoft.OperationalInsights/workspaces", "2020-03-01-preview") +let tables = + ResourceType("Microsoft.OperationalInsights/workspaces/tables", "2023-09-01") + +type Plan = + | Analytics of RetentionInDays: int option + | Auxiliary + | Basic + + member this.ArmValue = + match this with + | Analytics _ -> "Analytics" + | Auxiliary -> "Auxiliary" + | Basic -> "Basic" + + member this.RetentionInDays = + match this with + | Analytics days -> + match days with + | Some d -> Some d + | None -> Some -1 + | Auxiliary -> None + | Basic -> None + +type Table = { + Name: ResourceName + Plan: Plan + Columns: Column list + TotalRetentionInDays: int option + LogAnalyticsWorkspace: ResourceId +} with + + interface IArmResource with + member this.ResourceId = + tables.resourceId (this.LogAnalyticsWorkspace.Name / this.Name) + + member this.JsonModel = {| + tables.Create(this.LogAnalyticsWorkspace.Name / this.Name, dependsOn = [ this.LogAnalyticsWorkspace ]) with + properties = {| + plan = this.Plan.ArmValue + retentionInDays = this.Plan.RetentionInDays |> Option.toNullable + totalRetentionInDays = this.TotalRetentionInDays |> Option.defaultValue -1 + schema = {| + name = this.Name.Value + columns = + this.Columns + |> List.map (fun c -> {| + name = c.Name + ``type`` = c.Type.ArmValue + |}) + |} + |} + |} + type Workspace = { Name: ResourceName Location: Location diff --git a/src/Farmer/Arm/Monitor.fs b/src/Farmer/Arm/Monitor.fs index 14e6ec1af..cd9013b85 100644 --- a/src/Farmer/Arm/Monitor.fs +++ b/src/Farmer/Arm/Monitor.fs @@ -19,6 +19,7 @@ type DataCollectionEndpoint = { member this.JsonModel = {| dataCollectionEndpoints.Create(this.Name, this.Location, tags = this.Tags) with kind = string this.OsType + properties = {| |} |} let dataCollectionRules = @@ -64,16 +65,23 @@ type Stream = | Perf -> "Microsoft-Perf" | Syslog -> "Microsoft-Syslog" | WindowsEvent -> "Microsoft-WindowsEvent" - | CustomStream name -> name + | CustomStream tableName -> $"Custom-{tableName}_CL" type DataFlow = { Destinations: string list Streams: Stream list + TransformKQL: string option + OutputStream: Stream option } with member this.ToArmJson = {| destinations = this.Destinations streams = this.Streams |> List.map Stream.Print + transformKql = this.TransformKQL |> Option.defaultValue Unchecked.defaultof<_> + outputStream = + this.OutputStream + |> Option.map Stream.Print + |> Option.defaultValue Unchecked.defaultof<_> |} module Destinations = @@ -92,24 +100,48 @@ module Destinations = accountResourceId = this.AccountResourceId.Eval() |} + type LogAnalytics = { + WorkspaceResourceId: ResourceId + Name: ResourceName + } with + + static member Default = { + WorkspaceResourceId = ResourceId.Empty + Name = ResourceName.Empty + } + + member this.ToArmJson = {| + workspaceResourceId = this.WorkspaceResourceId.Eval() + name = this.Name.Value + |} + type Destination = { MonitoringAccounts: (MonitoringAccount list) option + LogAnalytics: (LogAnalytics list) option } with - static member Default = { MonitoringAccounts = None } + static member Default = { + MonitoringAccounts = None + LogAnalytics = None + } let ToArmJson (destinations: Destination) = {| monitoringAccounts = destinations.MonitoringAccounts |> Option.map (List.map (fun d -> d.ToArmJson)) |> Option.defaultValue Unchecked.defaultof<_> + logAnalytics = + destinations.LogAnalytics + |> Option.map (List.map (fun d -> d.ToArmJson)) + |> Option.defaultValue Unchecked.defaultof<_> |} type DataCollectionRule = { Name: ResourceName - OsType: OS + OsType: OS option Location: Location Endpoint: ResourceId + StreamDeclarations: Map DataFlows: (DataFlow list) option DataSources: DataSources.DataSource option Destinations: Destinations.Destination option @@ -125,17 +157,31 @@ type DataCollectionRule = { {| dataCollectionRules.Create(this.Name, this.Location, dependencies, this.Tags) with - kind = string this.OsType + kind = this.OsType |> Option.map string |> Option.defaultValue Unchecked.defaultof<_> properties = {| dataCollectionEndpointId = this.Endpoint.Eval() + streamDeclarations = + this.StreamDeclarations + |> Map.toList + |> List.map (fun (stream, columns) -> + Stream.Print stream, + {| + columns = + columns + |> List.map (fun col -> {| + name = col.Name + ``type`` = col.Type.ArmValue + |}) + |}) + |> Map.ofList dataFlows = this.DataFlows |> Option.map (List.map (fun flow -> flow.ToArmJson)) |> Option.defaultValue Unchecked.defaultof<_> dataSources = this.DataSources - |> Option.map DataSources.ToArmJson - |> Option.defaultValue Unchecked.defaultof<_> + |> Option.defaultValue DataSources.DataSource.Default + |> DataSources.ToArmJson destinations = this.Destinations |> Option.map Destinations.ToArmJson diff --git a/src/Farmer/Builders/Builders.AzureMonitor.fs b/src/Farmer/Builders/Builders.AzureMonitor.fs index d142e6cda..c87973ef7 100644 --- a/src/Farmer/Builders/Builders.AzureMonitor.fs +++ b/src/Farmer/Builders/Builders.AzureMonitor.fs @@ -61,21 +61,32 @@ type DataSourceConfig = type DestinationConfig = | MonitoringAccounts of MonitoringAccount list + | LogAnalytics of LogAnalytics list static member BuildConfig(destinations: DestinationConfig list) : Destinations.Destination = { MonitoringAccounts = destinations |> List.tryFind (function - | MonitoringAccounts _ -> true) + | MonitoringAccounts _ -> true + | _ -> false) |> function | Some(MonitoringAccounts accounts) -> Some accounts - | None -> None + | _ -> None + LogAnalytics = + destinations + |> List.tryFind (function + | LogAnalytics _ -> true + | _ -> false) + |> function + | Some(LogAnalytics workspaces) -> Some workspaces + | _ -> None } type DataCollectionRuleConfig = { Name: ResourceName - OsType: OS + OsType: OS option Endpoint: ResourceId + StreamDeclarations: (Stream * Column list) list DataFlows: (DataFlow list) option DataSources: DataSourceConfig list Destinations: DestinationConfig list @@ -92,6 +103,7 @@ type DataCollectionRuleConfig = { OsType = this.OsType Location = location Endpoint = this.Endpoint + StreamDeclarations = this.StreamDeclarations |> Map.ofList DataFlows = this.DataFlows DataSources = match this.DataSources with @@ -109,8 +121,9 @@ type DataCollectionRuleConfig = { type DataCollectionRuleBuilder() = member _.Yield _ = { Name = ResourceName.Empty - OsType = Linux + OsType = None Endpoint = ResourceId.Empty + StreamDeclarations = [] DataFlows = None DataSources = [] Destinations = [] @@ -129,12 +142,19 @@ type DataCollectionRuleBuilder() = /// Sets the kind for the data collection rule (Windows or Linux). [] - member _.OsType(state: DataCollectionRuleConfig, osType) = { state with OsType = osType } + member _.OsType(state: DataCollectionRuleConfig, osType) = { state with OsType = Some osType } /// Sets the endpoint for the data collection rule. [] member _.Endpoint(state: DataCollectionRuleConfig, endpoint) = { state with Endpoint = endpoint } + /// Sets the stream declarations for the data collection rule. + [] + member _.StreamDeclarations(state: DataCollectionRuleConfig, streamDeclarations) = { + state with + StreamDeclarations = streamDeclarations + } + /// Sets the data flows for the data collection rule. [] member _.DataFlows(state: DataCollectionRuleConfig, dataFlows) = { diff --git a/src/Farmer/Builders/Builders.ContainerApps.fs b/src/Farmer/Builders/Builders.ContainerApps.fs index c9e07b037..2b2b8f3f0 100644 --- a/src/Farmer/Builders/Builders.ContainerApps.fs +++ b/src/Farmer/Builders/Builders.ContainerApps.fs @@ -108,6 +108,7 @@ type ContainerEnvironmentConfig = { IngestionSupport = None QuerySupport = None DailyCap = None + CustomTables = [] Tags = Map.empty } :> IBuilder diff --git a/src/Farmer/Builders/Builders.LogAnalytics.fs b/src/Farmer/Builders/Builders.LogAnalytics.fs index d03274590..be6240991 100644 --- a/src/Farmer/Builders/Builders.LogAnalytics.fs +++ b/src/Farmer/Builders/Builders.LogAnalytics.fs @@ -9,12 +9,31 @@ let private (|InBounds|OutOfBounds|) days = elif days > 730 then OutOfBounds days else InBounds days +type TableConfig = { + Name: ResourceName + Plan: Plan + Columns: Column list + TotalRetentionInDays: int option +} with + + member this.BuildResources logAnalyticsWorkspace = [ + { + Name = ResourceName $"{this.Name.Value}_CL" + Plan = this.Plan + Columns = this.Columns + TotalRetentionInDays = this.TotalRetentionInDays + LogAnalyticsWorkspace = logAnalyticsWorkspace + } + :> IArmResource + ] + type WorkspaceConfig = { Name: ResourceName RetentionPeriod: int option IngestionSupport: FeatureFlag option QuerySupport: FeatureFlag option DailyCap: int option + CustomTables: TableConfig list Tags: Map } with @@ -37,6 +56,8 @@ type WorkspaceConfig = { DailyCap = this.DailyCap Tags = this.Tags } + for table in this.CustomTables do + yield! table.BuildResources (this :> IBuilder).ResourceId ] type WorkspaceBuilder() = @@ -46,6 +67,7 @@ type WorkspaceBuilder() = DailyCap = None IngestionSupport = None QuerySupport = None + CustomTables = [] Tags = Map.empty } @@ -87,6 +109,13 @@ type WorkspaceBuilder() = [] member _.DailyCap(state: WorkspaceConfig, cap) = { state with DailyCap = Some cap } + /// Adds tables to the Log Analytics workspace. + [] + member _.CustomTables(state: WorkspaceConfig, customTables: TableConfig list) = { + state with + CustomTables = customTables + } + interface ITaggable with member _.Add state tags = { state with diff --git a/src/Farmer/Common.fs b/src/Farmer/Common.fs index d82be4ae8..741071ef7 100644 --- a/src/Farmer/Common.fs +++ b/src/Farmer/Common.fs @@ -1,7 +1,28 @@ -namespace Farmer +namespace Farmer open System +type ColumnType = + | Boolean + | DateTime + | Dynamic + | Int + | Long + | Real + | String + + member this.ArmValue = + match this with + | Boolean -> "boolean" + | DateTime -> "datetime" + | Dynamic -> "dynamic" + | Int -> "int" + | Long -> "long" + | Real -> "real" + | String -> "string" + +type Column = { Name: string; Type: ColumnType } + type NonEmptyList<'T> = private | NonEmptyList of List<'T> diff --git a/src/Tests/DataCollection.fs b/src/Tests/DataCollection.fs index ab4fd9c50..31333bd39 100644 --- a/src/Tests/DataCollection.fs +++ b/src/Tests/DataCollection.fs @@ -42,6 +42,8 @@ let tests = { Streams = [ (CustomStream "Microsoft-PrometheusMetrics") ] Destinations = [ "Account1" ] + TransformKQL = None + OutputStream = None } ] diff --git a/src/Tests/Gallery.fs b/src/Tests/Gallery.fs index da7d73ebc..8d373041a 100644 --- a/src/Tests/Gallery.fs +++ b/src/Tests/Gallery.fs @@ -1,11 +1,11 @@ module Gallery -open System open Expecto open Farmer open Farmer.Builders open Farmer.Arm.Gallery open Newtonsoft.Json.Linq +open System let tests = testList "Image Gallery" [ diff --git a/src/Tests/LogAnalytics.fs b/src/Tests/LogAnalytics.fs index ebe24bb71..f7910b2b7 100644 --- a/src/Tests/LogAnalytics.fs +++ b/src/Tests/LogAnalytics.fs @@ -1,14 +1,14 @@ -module LogAnalytics +module LogAnalytics open Expecto open Farmer open Farmer.Arm open Farmer.Builders -open Farmer.Helpers open Microsoft.Azure.Management.OperationalInsights open Microsoft.Azure.Management.OperationalInsights.Models open Microsoft.Rest open System +open Newtonsoft.Json.Linq let dummyClient = new OperationalInsightsManagementClient(Uri "http://management.azure.com", TokenCredentials "NotNullOrWhiteSpace") @@ -41,6 +41,101 @@ let tests = Expect.equal workspace.RetentionInDays (Nullable 30) "Incorrect Retention In Days" } + test "Table created under workspace resource" { + let logging = logAnalytics { + name "log-analytics" + + custom_tables [ + { + Name = ResourceName "MyTable" + Plan = Analytics(Some 1) + Columns = [ + { + Name = "TimeGenerated" + Type = ColumnType.DateTime + } + { + Name = "Event" + Type = ColumnType.Dynamic + } + ] + TotalRetentionInDays = Some 2 + } + ] + } + + let deployment = arm { add_resource logging } + + let table = + deployment.Template.Resources + |> List.tryFind (fun r -> + r.ResourceId.Name.Value = "log-analytics" + && not (r.ResourceId.Segments |> List.isEmpty) + && (r.ResourceId.Segments |> List.exactlyOne).Value = "MyTable_CL") + |> Option.map (fun t -> t :?> Farmer.Arm.LogAnalytics.Table) + + Expect.equal (table.Value.Columns.Length) 2 "Incorrect number of columns in table" + Expect.equal (table.Value.Columns[0].Name) "TimeGenerated" "Incorrect first column name" + Expect.equal (table.Value.Columns[0].Type) ColumnType.DateTime "Incorrect first column type" + Expect.equal (table.Value.Columns[1].Name) "Event" "Incorrect second column name" + Expect.equal (table.Value.Columns[1].Type) ColumnType.Dynamic "Incorrect second column type" + Expect.equal (table.Value.TotalRetentionInDays) (Some 2) "Incorrect total retention in days" + Expect.equal (table.Value.Plan.ArmValue) "Analytics" "Incorrect plan type" + Expect.equal (table.Value.Plan.RetentionInDays) (Some 1) "Incorrect plan retention in days" + + Expect.equal + (table.Value.LogAnalyticsWorkspace.Name.Value) + "log-analytics" + "Incorrect workspace name in table resource" + } + + test "Table JSON emitted correctly" { + let logging = logAnalytics { + name "log-analytics" + + custom_tables [ + { + Name = ResourceName "MyTable" + Plan = Analytics(Some 1) + Columns = [ + { + Name = "TimeGenerated" + Type = ColumnType.DateTime + } + { + Name = "Event" + Type = ColumnType.Dynamic + } + ] + TotalRetentionInDays = Some 2 + } + ] + } + + let deployment = arm { add_resource logging } + let jsonTemplate = deployment.Template |> Writer.toJson + let jobj = JObject.Parse jsonTemplate + let tableJson = jobj["resources"][1] + Expect.equal (tableJson["type"] |> string) LogAnalytics.tables.Type "Incorrect resource type" + Expect.equal (tableJson["apiVersion"] |> string) LogAnalytics.tables.ApiVersion "Incorrect api version" + + Expect.equal + (tableJson["dependsOn"][0] |> string) + "[resourceId('Microsoft.OperationalInsights/workspaces', 'log-analytics')]" + "Incorrect dependsOn" + + Expect.equal (tableJson["name"] |> string) "log-analytics/MyTable_CL" "Incorrect resource name" + Expect.equal (tableJson["properties"]["plan"] |> string) "Analytics" "Incorrect plan type" + Expect.equal (tableJson["properties"]["retentionInDays"] |> int) 1 "Incorrect plan retention in days" + Expect.equal (tableJson["properties"]["totalRetentionInDays"] |> int) 2 "Incorrect total retention in days" + Expect.equal (tableJson["properties"].["schema"].["name"] |> string) "MyTable_CL" "Incorrect table name" + let columns = tableJson["properties"].["schema"].["columns"] + Expect.equal (columns.[0]["name"] |> string) "TimeGenerated" "Incorrect first column name" + Expect.equal (columns.[0]["type"] |> string) "datetime" "Incorrect first column type" + Expect.equal (columns.[1]["name"] |> string) "Event" "Incorrect second column name" + Expect.equal (columns.[1]["type"] |> string) "dynamic" "Incorrect second column type" + } + test "Ingestion and Query are disabled by default" { let workspace = logAnalytics { name "" } |> asAzureResource diff --git a/src/Tests/PostgreSQL.fs b/src/Tests/PostgreSQL.fs index 4335b2998..91f76077e 100644 --- a/src/Tests/PostgreSQL.fs +++ b/src/Tests/PostgreSQL.fs @@ -2,13 +2,12 @@ module PostgreSQL #nowarn "0044" // disable obsolete warning as error - needed -open System - open Expecto open Farmer open Farmer.Arm open Farmer.Builders open Farmer.PostgreSQL +open System type PostgresSku = { name: string