Custom log ingestion (tables, DCR destination etc)

docs/content/api-overview/resources/loganalytics.md

@@ -10,8 +10,9 @@ weight: 12
 The Log Analytics builder is used to create Work space instances.
 
 - Log Analytics (`Microsoft.OperationalInsights/workspaces`)
+- Tables (`Microsoft.OperationalInsights/workspaces/tables`)
 
-#### Builder Keywords
+#### Log Analytics Builder Keywords
 
 | Keyword          | Purpose                                                         |
 | ---------------- | --------------------------------------------------------------- |
@@ -20,9 +21,19 @@ The Log Analytics builder is used to create Work space instances.
 | enable_ingestion | Enables ingestion network traffic.                              |
 | enable_query     | Enables query network traffic.                                  |
 | daily_cap        | Specifies an upper limit on the amount of data to ingest daily. |
+| tables           | Defines tables to be created in the workspace.                  |
 | add_tags         | Adds a set of tags to the resource                              |
 | add_tag          | Adds a tag to the resource                                      |
 
+#### Table Builder Keywords
+
+| Keyword               | Purpose                                                                                                                               |
+| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------|
+| Name                  | Sets the name of the table.                                                                                                           |
+| Plan                  | Sets the table plan. Analytics plans can set a retention period between 4 and 730. If ommited will default to the workspace retention.|
+| Columns               | Sets the columns of the table. Each column has a name and type.                                                                       |
+| TotalRetentionInDays  | Sets the total retention period for the table in days, between 4 and 4383. If ommited will default to the Plan retention.             |
+
 #### Configuration Members
 
 | Member | Purpose |
@@ -42,6 +53,17 @@ let myAnalytics = logAnalytics {
     enable_ingestion
     enable_query
     daily_cap 5<Gb>
+    tables [
+        {
+            Name = ResourceName "Serilog"
+            Plan = Analytics (Some 30<Days>)
+            Columns = [
+                { Name = "TimeGenerated"; Type = "datetime" }
+                { Name = "Event"; Type = "dynamic" }
+            ]
+            TotalRetentionInDays = None
+        }
+    ]
     add_tag "tag1" "myTestResourceFarmer"
 }
 

src/Farmer/Arm/LogAnalytics.fs

@@ -6,6 +6,59 @@ open Farmer
 let workspaces =
     ResourceType("Microsoft.OperationalInsights/workspaces", "2020-03-01-preview")
 
+let tables =
+    ResourceType("Microsoft.OperationalInsights/workspaces/tables", "2023-09-01")
+
+type Plan =
+    | Analytics of RetentionInDays: int<Days> option
+    | Auxiliary
+    | Basic
+
+    member this.ArmValue =
+        match this with
+        | Analytics _ -> "Analytics"
+        | Auxiliary -> "Auxiliary"
+        | Basic -> "Basic"
+
+    member this.RetentionInDays =
+        match this with
+        | Analytics days ->
+            match days with
+            | Some d -> Some d
+            | None -> Some -1<Days>
+        | Auxiliary -> None
+        | Basic -> None
+
+type Table = {
+    Name: ResourceName
+    Plan: Plan
+    Columns: Column list
+    TotalRetentionInDays: int<Days> option
+    LogAnalyticsWorkspace: ResourceId
+} with
+
+    interface IArmResource with
+        member this.ResourceId =
+            tables.resourceId (this.LogAnalyticsWorkspace.Name / this.Name)
+
+        member this.JsonModel = {|
+            tables.Create(this.LogAnalyticsWorkspace.Name / this.Name, dependsOn = [ this.LogAnalyticsWorkspace ]) with
+                properties = {|
+                    plan = this.Plan.ArmValue
+                    retentionInDays = this.Plan.RetentionInDays |> Option.toNullable
+                    totalRetentionInDays = this.TotalRetentionInDays |> Option.defaultValue -1<Days>
+                    schema = {|
+                        name = this.Name.Value
+                        columns =
+                            this.Columns
+                            |> List.map (fun c -> {|
+                                name = c.Name
+                                ``type`` = c.Type.ArmValue
+                            |})
+                    |}
+                |}
+        |}
+
 type Workspace = {
     Name: ResourceName
     Location: Location

src/Farmer/Arm/Monitor.fs

@@ -19,6 +19,7 @@ type DataCollectionEndpoint = {
         member this.JsonModel = {|
             dataCollectionEndpoints.Create(this.Name, this.Location, tags = this.Tags) with
                 kind = string this.OsType
+                properties = {| |}
         |}
 
 let dataCollectionRules =
@@ -64,16 +65,23 @@ type Stream =
         | Perf -> "Microsoft-Perf"
         | Syslog -> "Microsoft-Syslog"
         | WindowsEvent -> "Microsoft-WindowsEvent"
-        | CustomStream name -> name
+        | CustomStream tableName -> $"Custom-{tableName}_CL"
 
 type DataFlow = {
     Destinations: string list
     Streams: Stream list
+    TransformKQL: string option
+    OutputStream: Stream option
 } with
 
     member this.ToArmJson = {|
         destinations = this.Destinations
         streams = this.Streams |> List.map Stream.Print
+        transformKql = this.TransformKQL |> Option.defaultValue Unchecked.defaultof<_>
+        outputStream =
+            this.OutputStream
+            |> Option.map Stream.Print
+            |> Option.defaultValue Unchecked.defaultof<_>
     |}
 
 module Destinations =
@@ -92,24 +100,48 @@ module Destinations =
             accountResourceId = this.AccountResourceId.Eval()
         |}
 
+    type LogAnalytics = {
+        WorkspaceResourceId: ResourceId
+        Name: ResourceName
+    } with
+
+        static member Default = {
+            WorkspaceResourceId = ResourceId.Empty
+            Name = ResourceName.Empty
+        }
+
+        member this.ToArmJson = {|
+            workspaceResourceId = this.WorkspaceResourceId.Eval()
+            name = this.Name.Value
+        |}
+
     type Destination = {
         MonitoringAccounts: (MonitoringAccount list) option
+        LogAnalytics: (LogAnalytics list) option
     } with
 
-        static member Default = { MonitoringAccounts = None }
+        static member Default = {
+            MonitoringAccounts = None
+            LogAnalytics = None
+        }
 
     let ToArmJson (destinations: Destination) = {|
         monitoringAccounts =
             destinations.MonitoringAccounts
             |> Option.map (List.map (fun d -> d.ToArmJson))
             |> Option.defaultValue Unchecked.defaultof<_>
+        logAnalytics =
+            destinations.LogAnalytics
+            |> Option.map (List.map (fun d -> d.ToArmJson))
+            |> Option.defaultValue Unchecked.defaultof<_>
     |}
 
 type DataCollectionRule = {
     Name: ResourceName
-    OsType: OS
+    OsType: OS option
     Location: Location
     Endpoint: ResourceId
+    StreamDeclarations: Map<Stream, Column list>
     DataFlows: (DataFlow list) option
     DataSources: DataSources.DataSource option
     Destinations: Destinations.Destination option
@@ -125,17 +157,31 @@ type DataCollectionRule = {
 
             {|
                 dataCollectionRules.Create(this.Name, this.Location, dependencies, this.Tags) with
-                    kind = string this.OsType
+                    kind = this.OsType |> Option.map string |> Option.defaultValue Unchecked.defaultof<_>
                     properties = {|
                         dataCollectionEndpointId = this.Endpoint.Eval()
+                        streamDeclarations =
+                            this.StreamDeclarations
+                            |> Map.toList
+                            |> List.map (fun (stream, columns) ->
+                                Stream.Print stream,
+                                {|
+                                    columns =
+                                        columns
+                                        |> List.map (fun col -> {|
+                                            name = col.Name
+                                            ``type`` = col.Type.ArmValue
+                                        |})
+                                |})
+                            |> Map.ofList
                         dataFlows =
                             this.DataFlows
                             |> Option.map (List.map (fun flow -> flow.ToArmJson))
                             |> Option.defaultValue Unchecked.defaultof<_>
                         dataSources =
                             this.DataSources
-                            |> Option.map DataSources.ToArmJson
-                            |> Option.defaultValue Unchecked.defaultof<_>
+                            |> Option.defaultValue DataSources.DataSource.Default
+                            |> DataSources.ToArmJson
                         destinations =
                             this.Destinations
                             |> Option.map Destinations.ToArmJson

src/Farmer/Builders/Builders.AzureMonitor.fs

@@ -61,21 +61,32 @@ type DataSourceConfig =
 
 type DestinationConfig =
     | MonitoringAccounts of MonitoringAccount list
+    | LogAnalytics of LogAnalytics list
 
     static member BuildConfig(destinations: DestinationConfig list) : Destinations.Destination = {
         MonitoringAccounts =
             destinations
             |> List.tryFind (function
-                | MonitoringAccounts _ -> true)
+                | MonitoringAccounts _ -> true
+                | _ -> false)
             |> function
                 | Some(MonitoringAccounts accounts) -> Some accounts
-                | None -> None
+                | _ -> None
+        LogAnalytics =
+            destinations
+            |> List.tryFind (function
+                | LogAnalytics _ -> true
+                | _ -> false)
+            |> function
+                | Some(LogAnalytics workspaces) -> Some workspaces
+                | _ -> None
     }
 
 type DataCollectionRuleConfig = {
     Name: ResourceName
-    OsType: OS
+    OsType: OS option
     Endpoint: ResourceId
+    StreamDeclarations: (Stream * Column list) list
     DataFlows: (DataFlow list) option
     DataSources: DataSourceConfig list
     Destinations: DestinationConfig list
@@ -92,6 +103,7 @@ type DataCollectionRuleConfig = {
                 OsType = this.OsType
                 Location = location
                 Endpoint = this.Endpoint
+                StreamDeclarations = this.StreamDeclarations |> Map.ofList
                 DataFlows = this.DataFlows
                 DataSources =
                     match this.DataSources with
@@ -109,8 +121,9 @@ type DataCollectionRuleConfig = {
 type DataCollectionRuleBuilder() =
     member _.Yield _ = {
         Name = ResourceName.Empty
-        OsType = Linux
+        OsType = None
         Endpoint = ResourceId.Empty
+        StreamDeclarations = []
         DataFlows = None
         DataSources = []
         Destinations = []
@@ -129,12 +142,19 @@ type DataCollectionRuleBuilder() =
 
     /// Sets the kind for the data collection rule (Windows or Linux).
     [<CustomOperation "os_type">]
-    member _.OsType(state: DataCollectionRuleConfig, osType) = { state with OsType = osType }
+    member _.OsType(state: DataCollectionRuleConfig, osType) = { state with OsType = Some osType }
 
     /// Sets the endpoint for the data collection rule.
     [<CustomOperation "endpoint">]
     member _.Endpoint(state: DataCollectionRuleConfig, endpoint) = { state with Endpoint = endpoint }
 
+    /// Sets the stream declarations for the data collection rule.
+    [<CustomOperation "stream_declarations">]
+    member _.StreamDeclarations(state: DataCollectionRuleConfig, streamDeclarations) = {
+        state with
+            StreamDeclarations = streamDeclarations
+    }
+
     /// Sets the data flows for the data collection rule.
     [<CustomOperation "data_flows">]
     member _.DataFlows(state: DataCollectionRuleConfig, dataFlows) = {

src/Farmer/Builders/Builders.ContainerApps.fs

@@ -108,6 +108,7 @@ type ContainerEnvironmentConfig = {
                         IngestionSupport = None
                         QuerySupport = None
                         DailyCap = None
+                        CustomTables = []
                         Tags = Map.empty
                     }
                     :> IBuilder

src/Farmer/Builders/Builders.LogAnalytics.fs

@@ -9,12 +9,31 @@ let private (|InBounds|OutOfBounds|) days =
     elif days > 730<Days> then OutOfBounds days
     else InBounds days
 
+type TableConfig = {
+    Name: ResourceName
+    Plan: Plan
+    Columns: Column list
+    TotalRetentionInDays: int<Days> option
+} with
+
+    member this.BuildResources logAnalyticsWorkspace = [
+        {
+            Name = ResourceName $"{this.Name.Value}_CL"
+            Plan = this.Plan
+            Columns = this.Columns
+            TotalRetentionInDays = this.TotalRetentionInDays
+            LogAnalyticsWorkspace = logAnalyticsWorkspace
+        }
+        :> IArmResource
+    ]
+
 type WorkspaceConfig = {
     Name: ResourceName
     RetentionPeriod: int<Days> option
     IngestionSupport: FeatureFlag option
     QuerySupport: FeatureFlag option
     DailyCap: int<Gb> option
+    CustomTables: TableConfig list
     Tags: Map<string, string>
 } with
 
@@ -37,6 +56,8 @@ type WorkspaceConfig = {
                 DailyCap = this.DailyCap
                 Tags = this.Tags
             }
+            for table in this.CustomTables do
+                yield! table.BuildResources (this :> IBuilder).ResourceId
         ]
 
 type WorkspaceBuilder() =
@@ -46,6 +67,7 @@ type WorkspaceBuilder() =
         DailyCap = None
         IngestionSupport = None
         QuerySupport = None
+        CustomTables = []
         Tags = Map.empty
     }
 
@@ -87,6 +109,13 @@ type WorkspaceBuilder() =
     [<CustomOperation "daily_cap">]
     member _.DailyCap(state: WorkspaceConfig, cap) = { state with DailyCap = Some cap }
 
+    /// Adds tables to the Log Analytics workspace.
+    [<CustomOperation "custom_tables">]
+    member _.CustomTables(state: WorkspaceConfig, customTables: TableConfig list) = {
+        state with
+            CustomTables = customTables
+    }
+
     interface ITaggable<WorkspaceConfig> with
         member _.Add state tags = {
             state with