diff --git a/.github/workflows/azure-sentinel-deploy-70b27887-856e-4f03-a7f0-2cec8af84042.ps1 b/.github/workflows/azure-sentinel-deploy-70b27887-856e-4f03-a7f0-2cec8af84042.ps1 new file mode 100644 index 000000000..a01e7a643 --- /dev/null +++ b/.github/workflows/azure-sentinel-deploy-70b27887-856e-4f03-a7f0-2cec8af84042.ps1 @@ -0,0 +1,650 @@ +## Globals ## +$CloudEnv = $Env:cloudEnv +$ResourceGroupName = $Env:resourceGroupName +$WorkspaceName = $Env:workspaceName +$WorkspaceId = $Env:workspaceId +$Directory = $Env:directory +$contentTypes = $Env:contentTypes +$contentTypeMapping = @{ + "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); + "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); + "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); + "Workbook"=@("Microsoft.Insights/workbooks"); + "CustomDetection"=@("Microsoft.XDR/customDetections"); +} +$sourceControlId = $Env:sourceControlId +$rootDirectory = $Env:rootDirectory +$githubAuthToken = $Env:githubAuthToken +$githubRepository = $Env:GITHUB_REPOSITORY +$branchName = $Env:branch +$smartDeployment = $Env:smartDeployment +$newResourceBranch = $branchName + "-sentinel-deployment" +$csvPath = "$rootDirectory\.sentinel\tracking_table_$sourceControlId.csv" +$configPath = "$rootDirectory\sentinel-deployment.config" +$global:localCsvTablefinal = @{} +$global:updatedCsvTable = @{} +$global:parameterFileMapping = @{} +$global:prioritizedContentFiles = @() +$global:excludeContentFiles = @() + +$guidPattern = '(\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)' +$namePattern = '([-\w\._\(\)]+)' +$sentinelResourcePatterns = @{ + "AnalyticsRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/alertRules/$namePattern" + "AutomationRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/automationRules/$namePattern" + "HuntingQuery" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" + "Parser" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" + "Playbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Logic/workflows/$namePattern" + "Workbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Insights/workbooks/$namePattern" + "CustomDetection" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.XDR/customDetections/$namePattern" +} + +if ([string]::IsNullOrEmpty($contentTypes)) { + $contentTypes = "AnalyticsRule" +} + +$metadataFilePath = "metadata.json" +@" +{ + "`$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "parentResourceId": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "sourceControlId": { + "type": "string" + }, + "workspace": { + "type": "string" + }, + "contentId": { + "type": "string" + }, + "customVersion": { + "type": "string" + } + }, + "variables": { + "metadataName": "[concat(toLower(parameters('kind')), '-', parameters('contentId'))]" + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('metadataName'))]", + "properties": { + "parentId": "[parameters('parentResourceId')]", + "kind": "[parameters('kind')]", + "customVersion": "[parameters('customVersion')]", + "source": { + "kind": "SourceRepository", + "name": "Repositories", + "sourceId": "[parameters('sourceControlId')]" + } + } + } + ] +} +"@ | Out-File -FilePath $metadataFilePath + +$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } +$MaxRetries = 3 +$secondsBetweenAttempts = 5 + +#Converts hashtable to string that can be set as content when pushing csv file +function ConvertTableToString { + $output = "FileName, CommitSha`n" + $global:updatedCsvTable.GetEnumerator() | ForEach-Object { + $key = RelativePathWithBackslash $_.Key + $output += "{0},{1}`n" -f $key, $_.Value + } + return $output +} + +$header = @{ + "authorization" = "Bearer $githubAuthToken" +} + +#Gets all files and commit shas using Get Trees API +function GetGithubTree { + $branchResponse = AttemptInvokeRestMethod "Get" "https://api.github.com/repos/$githubRepository/branches/$branchName" $null $null 3 + $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" + $getTreeResponse = AttemptInvokeRestMethod "Get" $treeUrl $null $null 3 + return $getTreeResponse +} + +#Creates a table using the reponse from the tree api, creates a table +function GetCommitShaTable($getTreeResponse) { + $shaTable = @{} + $supportedExtensions = @(".json", ".bicep", ".bicepparam"); + $getTreeResponse.tree | ForEach-Object { + $truePath = AbsolutePathWithSlash $_.path + if ((([System.IO.Path]::GetExtension($_.path) -in $supportedExtensions)) -or ($truePath -eq $configPath)) + { + $shaTable.Add($truePath, $_.sha) + } + } + return $shaTable +} + +function PushCsvToRepo() { + $content = ConvertTableToString + $relativeCsvPath = RelativePathWithBackslash $csvPath + $resourceBranchExists = git ls-remote --heads "https://github.com/$githubRepository" $newResourceBranch | wc -l + + if ($resourceBranchExists -eq 0) { + git switch --orphan $newResourceBranch + git commit --allow-empty -m "Initial commit on orphan branch" + git push -u origin $newResourceBranch + New-Item -ItemType "directory" -Path ".sentinel" + } else { + git fetch > $null + git checkout $newResourceBranch + } + + Write-Output $content > $relativeCsvPath + git add $relativeCsvPath + git commit -m "Modified tracking table" + git push -u origin $newResourceBranch + git checkout $branchName +} + +function ReadCsvToTable { + $csvTable = Import-Csv -Path $csvPath + $HashTable=@{} + foreach($r in $csvTable) + { + $key = AbsolutePathWithSlash $r.FileName + $HashTable[$key]=$r.CommitSha + } + return $HashTable +} + +function AttemptInvokeRestMethod($method, $url, $body, $contentTypes, $maxRetries) { + $Stoploop = $false + $retryCount = 0 + do { + try { + $result = Invoke-RestMethod -Uri $url -Method $method -Headers $header -Body $body -ContentType $contentTypes + $Stoploop = $true + } + catch { + if ($retryCount -gt $maxRetries) { + Write-Host "[Error] API call failed after $retryCount retries: $_" + $Stoploop = $true + } + else { + Write-Host "[Warning] API call failed: $_.`n Conducting retry #$retryCount." + Start-Sleep -Seconds 5 + $retryCount = $retryCount + 1 + } + } + } + While ($Stoploop -eq $false) + return $result +} + +function AttemptDeployMetadata($deploymentName, $resourceGroupName, $templateObject, $templateType, $paramFileType, $containsWorkspaceParam) { + $deploymentInfo = $null + try { + $deploymentInfo = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Ignore + } + catch { + Write-Host "[Warning] Unable to fetch deployment info for $deploymentName, no metadata was created for the resources in the file. Error: $_" + return + } + $deploymentInfo | Where-Object { $_.TargetResource -ne "" } | ForEach-Object { + $resource = $_.TargetResource + $sentinelContentKinds = GetContentKinds $resource + if ($sentinelContentKinds.Count -gt 0) { + $contentKind = ToContentKind $sentinelContentKinds $resource $templateObject + + if ($contentKind -eq "CustomDetection") { + Write-Host "[Info] Skipping metadata deployment for CustomDetection content." + return + } + + $contentId = $resource.Split("/")[-1] + $metadataCustomVersion = GetMetadataCustomVersion $templateType $paramFileType $containsWorkspaceParam + + $isSuccess = $false + $currentAttempt = 0 + + While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) + { + $currentAttempt ++ + Try + { + New-AzResourceGroupDeployment -Name "md-$deploymentName" -ResourceGroupName $ResourceGroupName -TemplateFile $metadataFilePath ` + -parentResourceId $resource ` + -kind $contentKind ` + -contentId $contentId ` + -sourceControlId $sourceControlId ` + -workspace $workspaceName ` + -customVersion $metadataCustomVersion ` + -ErrorAction Stop | Out-Host + Write-Host "[Info] Created metadata for $contentKind with parent resource id $resource" + $isSuccess = $true + } + Catch [Exception] + { + $err = $_ + if (-not (IsRetryable "md-$deploymentName")) + { + Write-Host "[Warning] Failed to deploy metadata for $contentKind with parent resource id $resource with error: $err" + break + } + else + { + if ($currentAttempt -le $MaxRetries) + { + Write-Host "[Warning] Failed to deploy metadata for $contentKind with error: $err. Retrying in $secondsBetweenAttempts seconds..." + Start-Sleep -Seconds $secondsBetweenAttempts + } + else + { + Write-Host "[Warning] Failed to deploy metadata for $contentKind after $currentAttempt attempts with error: $err" + } + } + } + } + } + } +} + +function GetMetadataCustomVersion($templateType, $paramFileType, $containsWorkspaceParam){ + $customVersion = $templateType + "-" + $paramFileType + if($containsWorkspaceParam){ + $customVersion += "-WorkspaceParam" + } + if($smartDeployment -eq "true"){ + $customVersion += "-SmartTracking" + } + return $customVersion +} + +function GetContentKinds($resource) { + return $sentinelResourcePatterns.Keys | Where-Object { $resource -match $sentinelResourcePatterns[$_] } +} + +function ToContentKind($contentKinds, $resource, $templateObject) { + if ($contentKinds.Count -eq 1) { + return $contentKinds + } + if ($null -ne $resource -and $resource.Contains('savedSearches')) { + if ($templateObject.resources.properties.Category -eq "Hunting Queries") { + return "HuntingQuery" + } + return "Parser" + } + return $null +} + +function IsValidTemplate($path, $templateObject, $parameterFile) { + Try { + if (DoesContainWorkspaceParam $templateObject) { + if ($parameterFile) { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile -workspace $WorkspaceName + } + else { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName + } + } + else { + if ($parameterFile) { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile + } else { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path + } + } + + return $true + } + Catch { + Write-Host "[Warning] The file $path is not valid: $_" + return $false + } +} + +function IsRetryable($deploymentName) { + $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" + Try { + $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop + return $retryableStatusCodes -contains $deploymentResult.StatusCode + } + Catch { + return $false + } +} + +function IsValidResourceType($template) { + try { + $isAllowedResources = $true + $template.resources | ForEach-Object { + $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources + } + } + catch { + Write-Host "[Error] Failed to check valid resource type." + $isAllowedResources = $false + } + return $isAllowedResources +} + +function DoesContainWorkspaceParam($templateObject) { + $templateObject.parameters.PSobject.Properties.Name -contains "workspace" +} + +function AttemptDeployment($path, $parameterFile, $deploymentName, $templateObject, $templateType) { + Write-Host "[Info] Deploying $path with deployment name $deploymentName" + + $isValid = IsValidTemplate $path $templateObject $parameterFile + if (-not $isValid) { + Write-Host "[Error] Not deploying $path since the template is not valid" + return $false + } + $isSuccess = $false + $currentAttempt = 0 + While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) + { + $currentAttempt ++ + Try + { + Write-Host "[Info] Deploy $path with parameter file: [$parameterFile]" + $paramFileType = if(!$parameterFile) {"NoParam"} elseif($parameterFile -like "*.bicepparam") {"BicepParam"} else {"JsonParam"} + $containsWorkspaceParam = DoesContainWorkspaceParam $templateObject + if ($containsWorkspaceParam) + { + if ($parameterFile) { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host + } + else + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host + } + } + else + { + if ($parameterFile) { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host + } + else + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host + } + } + AttemptDeployMetadata $deploymentName $ResourceGroupName $templateObject $templateType $paramFileType $containsWorkspaceParam + + $isSuccess = $true + } + Catch [Exception] + { + $err = $_ + if (-not (IsRetryable $deploymentName)) + { + Write-Host "[Warning] Failed to deploy $path with error: $err" + break + } + else + { + if ($currentAttempt -le $MaxRetries) + { + Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." + Start-Sleep -Seconds $secondsBetweenAttempts + } + else + { + Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" + } + } + } + } + return $isSuccess +} + +function GenerateDeploymentName() { + $randomId = [guid]::NewGuid() + return "Sentinel_Deployment_$randomId" +} + +#Load deployment configuration +function LoadDeploymentConfig() { + Write-Host "[Info] load the deployment configuration from [$configPath]" + $global:parameterFileMapping = @{} + $global:prioritizedContentFiles = @() + $global:excludeContentFiles = @() + try { + if (Test-Path $configPath) { + $deployment_config = Get-Content $configPath | Out-String | ConvertFrom-Json + $parameterFileMappings = @{} + if ($deployment_config.parameterfilemappings) { + $deployment_config.parameterfilemappings.psobject.properties | ForEach { $parameterFileMappings[$_.Name] = $_.Value } + } + $key = ($parameterFileMappings.Keys | ? { $_ -eq $workspaceId }) + if ($null -ne $key) { + $parameterFileMappings[$key].psobject.properties | ForEach { $global:parameterFileMapping[$_.Name] = $_.Value } + } + if ($deployment_config.prioritizedcontentfiles) { + $global:prioritizedContentFiles = $deployment_config.prioritizedcontentfiles + } + $excludeList = $global:parameterFileMapping.Values + $global:prioritizedcontentfiles + if ($deployment_config.excludecontentfiles) { + $excludeList = $excludeList + $deployment_config.excludecontentfiles + } + $global:excludeContentFiles = $excludeList | Where-Object { Test-Path (AbsolutePathWithSlash $_) } + } + } + catch { + Write-Host "[Warning] An error occurred while trying to load deployment configuration." + Write-Host "Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function filterContentFile($fullPath) { + $temp = RelativePathWithBackslash $fullPath + return $global:excludeContentFiles | Where-Object {$temp.StartsWith($_, 'CurrentCultureIgnoreCase')} +} + +function RelativePathWithBackslash($absolutePath) { + return $absolutePath.Replace($rootDirectory + "\", "").Replace("\", "/") +} + +function AbsolutePathWithSlash($relativePath) { + return Join-Path -Path $rootDirectory -ChildPath $relativePath +} + +#resolve parameter file name, return $null if there is none. +function GetParameterFile($path) { + if ($path.Length -eq 0) { + return $null + } + + $index = RelativePathWithBackslash $path + $key = ($global:parameterFileMapping.Keys | Where-Object { $_ -eq $index }) + if ($key) { + $mappedParameterFile = AbsolutePathWithSlash $global:parameterFileMapping[$key] + if (Test-Path $mappedParameterFile) { + return $mappedParameterFile + } + } + + $extension = [System.IO.Path]::GetExtension($path) + if ($extension -ne ".json" -and $extension -ne ".bicep") { + return $null + } + + $parameterFilePrefix = $path.Substring(0, $path.Length - $extension.Length) + + # Check for workspace-specific parameter file + if ($extension -eq ".bicep") { + $workspaceParameterFile = $parameterFilePrefix + "-$WorkspaceId.bicepparam" + if (Test-Path $workspaceParameterFile) { + return $workspaceParameterFile + } + } + + $workspaceParameterFile = $parameterFilePrefix + ".parameters-$WorkspaceId.json" + if (Test-Path $workspaceParameterFile) { + return $workspaceParameterFile + } + + # Check for parameter file + if ($extension -eq ".bicep") { + $defaultParameterFile = $parameterFilePrefix + ".bicepparam" + Write-Host "Default parameter file: $defaultParameterFile" + if (Test-Path $defaultParameterFile) { + return $defaultParameterFile + } + } + + $defaultParameterFile = $parameterFilePrefix + ".parameters.json" + Write-Host "Default parameter file: $defaultParameterFile" + if (Test-Path $defaultParameterFile) { + return $defaultParameterFile + } + + return $null +} + +function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { + Write-Host "Starting Deployment for Files in path: $Directory" + if (Test-Path -Path $Directory) + { + $totalFiles = 0; + $totalFailed = 0; + $iterationList = @() + $global:prioritizedContentFiles | ForEach-Object { $iterationList += (AbsolutePathWithSlash $_) } + Get-ChildItem -Path $Directory -Recurse -Include *.bicep, *.json -exclude *metadata.json, *.parameters*.json, *.bicepparam, bicepconfig.json | + Where-Object { $null -eq ( filterContentFile $_.FullName ) } | + Select-Object -Property FullName | + ForEach-Object { $iterationList += $_.FullName } + $iterationList | ForEach-Object { + $path = $_ + Write-Host "[Info] Try to deploy $path" + if (-not (Test-Path $path)) { + Write-Host "[Warning] Skipping deployment for $path. The file doesn't exist." + return + } + + if ($path -like "*.bicep") { + $templateType = "Bicep" + $templateObject = bicep build $path --stdout | Out-String | ConvertFrom-Json + } else { + $templateType = "ARM" + $templateObject = Get-Content $path | Out-String | ConvertFrom-Json + } + + if (-not (IsValidResourceType $templateObject)) + { + Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." + return + } + $parameterFile = GetParameterFile $path + $result = SmartDeployment $fullDeploymentFlag $remoteShaTable $path $parameterFile $templateObject $templateType + if ($result.isSuccess -eq $false) { + $totalFailed++ + } + if (-not $result.skip) { + $totalFiles++ + } + if ($result.isSuccess -or $result.skip) { + $global:updatedCsvTable[$path] = $remoteShaTable[$path] + if ($parameterFile) { + $global:updatedCsvTable[$parameterFile] = $remoteShaTable[$parameterFile] + } + } + } + PushCsvToRepo + if ($totalFiles -gt 0 -and $totalFailed -gt 0) + { + $err = "$totalFailed of $totalFiles deployments failed." + Throw $err + } + } + else + { + Write-Output "[Warning] $Directory not found. nothing to deploy" + } +} + +function SmartDeployment($fullDeploymentFlag, $remoteShaTable, $path, $parameterFile, $templateObject, $templateType) { + try { + $skip = $false + $isSuccess = $null + if (!$fullDeploymentFlag) { + $existingSha = $global:localCsvTablefinal[$path] + $remoteSha = $remoteShaTable[$path] + $skip = (($existingSha) -and ($existingSha -eq $remoteSha)) + if ($skip -and $parameterFile) { + $existingShaForParameterFile = $global:localCsvTablefinal[$parameterFile] + $remoteShaForParameterFile = $remoteShaTable[$parameterFile] + $skip = (($existingShaForParameterFile) -and ($existingShaForParameterFile -eq $remoteShaForParameterFile)) + } + } + if (!$skip) { + $deploymentName = GenerateDeploymentName + $isSuccess = AttemptDeployment $path $parameterFile $deploymentName $templateObject $templateType + } + return @{ + skip = $skip + isSuccess = $isSuccess + } + } + catch { + Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function TryGetCsvFile { + if (Test-Path $csvPath) { + $global:localCsvTablefinal = ReadCsvToTable + Remove-Item -Path $csvPath + git add $csvPath + git commit -m "Removed tracking file and moved to new sentinel created branch" + git push origin $branchName + } + + $relativeCsvPath = RelativePathWithBackslash $csvPath + $resourceBranchExists = git ls-remote --heads "https://github.com/$githubRepository" $newResourceBranch | wc -l + + if ($resourceBranchExists -eq 1) { + git fetch > $null + git checkout $newResourceBranch + + if (Test-Path $relativeCsvPath) { + $global:localCsvTablefinal = ReadCsvToTable + } + git checkout $branchName + } +} + +function main() { + git config --global user.email "donotreply@microsoft.com" + git config --global user.name "Sentinel" + + TryGetCsvFile + LoadDeploymentConfig + $tree = GetGithubTree + $remoteShaTable = GetCommitShaTable $tree + + $existingConfigSha = $global:localCsvTablefinal[$configPath] + $remoteConfigSha = $remoteShaTable[$configPath] + $modifiedConfig = ($existingConfigSha -xor $remoteConfigSha) -or ($existingConfigSha -and $remoteConfigSha -and ($existingConfigSha -ne $remoteConfigSha)) + + if ($remoteConfigSha) { + $global:updatedCsvTable[$configPath] = $remoteConfigSha + } + + $fullDeploymentFlag = $modifiedConfig -or ($smartDeployment -eq "false") + Deployment $fullDeploymentFlag $remoteShaTable $tree +} + +main \ No newline at end of file diff --git a/.github/workflows/azure-sentinel-deploy-e7383813-bbb9-4fd8-b4dc-743beac9ee13.ps1 b/.github/workflows/azure-sentinel-deploy-e7383813-bbb9-4fd8-b4dc-743beac9ee13.ps1 new file mode 100644 index 000000000..2aaac50d2 --- /dev/null +++ b/.github/workflows/azure-sentinel-deploy-e7383813-bbb9-4fd8-b4dc-743beac9ee13.ps1 @@ -0,0 +1,642 @@ +## Globals ## +$CloudEnv = $Env:cloudEnv +$ResourceGroupName = $Env:resourceGroupName +$WorkspaceName = $Env:workspaceName +$WorkspaceId = $Env:workspaceId +$Directory = $Env:directory +$contentTypes = $Env:contentTypes +$contentTypeMapping = @{ + "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); + "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); + "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); + "Workbook"=@("Microsoft.Insights/workbooks"); +} +$sourceControlId = $Env:sourceControlId +$rootDirectory = $Env:rootDirectory +$githubAuthToken = $Env:githubAuthToken +$githubRepository = $Env:GITHUB_REPOSITORY +$branchName = $Env:branch +$smartDeployment = $Env:smartDeployment +$newResourceBranch = $branchName + "-sentinel-deployment" +$csvPath = "$rootDirectory\.sentinel\tracking_table_$sourceControlId.csv" +$configPath = "$rootDirectory\sentinel-deployment.config" +$global:localCsvTablefinal = @{} +$global:updatedCsvTable = @{} +$global:parameterFileMapping = @{} +$global:prioritizedContentFiles = @() +$global:excludeContentFiles = @() + +$guidPattern = '(\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)' +$namePattern = '([-\w\._\(\)]+)' +$sentinelResourcePatterns = @{ + "AnalyticsRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/alertRules/$namePattern" + "AutomationRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/automationRules/$namePattern" + "HuntingQuery" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" + "Parser" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" + "Playbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Logic/workflows/$namePattern" + "Workbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Insights/workbooks/$namePattern" +} + +if ([string]::IsNullOrEmpty($contentTypes)) { + $contentTypes = "AnalyticsRule" +} + +$metadataFilePath = "metadata.json" +@" +{ + "`$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "parentResourceId": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "sourceControlId": { + "type": "string" + }, + "workspace": { + "type": "string" + }, + "contentId": { + "type": "string" + }, + "customVersion": { + "type": "string" + } + }, + "variables": { + "metadataName": "[concat(toLower(parameters('kind')), '-', parameters('contentId'))]" + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('metadataName'))]", + "properties": { + "parentId": "[parameters('parentResourceId')]", + "kind": "[parameters('kind')]", + "customVersion": "[parameters('customVersion')]", + "source": { + "kind": "SourceRepository", + "name": "Repositories", + "sourceId": "[parameters('sourceControlId')]" + } + } + } + ] +} +"@ | Out-File -FilePath $metadataFilePath + +$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } +$MaxRetries = 3 +$secondsBetweenAttempts = 5 + +#Converts hashtable to string that can be set as content when pushing csv file +function ConvertTableToString { + $output = "FileName, CommitSha`n" + $global:updatedCsvTable.GetEnumerator() | ForEach-Object { + $key = RelativePathWithBackslash $_.Key + $output += "{0},{1}`n" -f $key, $_.Value + } + return $output +} + +$header = @{ + "authorization" = "Bearer $githubAuthToken" +} + +#Gets all files and commit shas using Get Trees API +function GetGithubTree { + $branchResponse = AttemptInvokeRestMethod "Get" "https://api.github.com/repos/$githubRepository/branches/$branchName" $null $null 3 + $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" + $getTreeResponse = AttemptInvokeRestMethod "Get" $treeUrl $null $null 3 + return $getTreeResponse +} + +#Creates a table using the reponse from the tree api, creates a table +function GetCommitShaTable($getTreeResponse) { + $shaTable = @{} + $supportedExtensions = @(".json", ".bicep", ".bicepparam"); + $getTreeResponse.tree | ForEach-Object { + $truePath = AbsolutePathWithSlash $_.path + if ((([System.IO.Path]::GetExtension($_.path) -in $supportedExtensions)) -or ($truePath -eq $configPath)) + { + $shaTable.Add($truePath, $_.sha) + } + } + return $shaTable +} + +function PushCsvToRepo() { + $content = ConvertTableToString + $relativeCsvPath = RelativePathWithBackslash $csvPath + $resourceBranchExists = git ls-remote --heads "https://github.com/$githubRepository" $newResourceBranch | wc -l + + if ($resourceBranchExists -eq 0) { + git switch --orphan $newResourceBranch + git commit --allow-empty -m "Initial commit on orphan branch" + git push -u origin $newResourceBranch + New-Item -ItemType "directory" -Path ".sentinel" + } else { + git fetch > $null + git checkout $newResourceBranch + } + + Write-Output $content > $relativeCsvPath + git add $relativeCsvPath + git commit -m "Modified tracking table" + git push -u origin $newResourceBranch + git checkout $branchName +} + +function ReadCsvToTable { + $csvTable = Import-Csv -Path $csvPath + $HashTable=@{} + foreach($r in $csvTable) + { + $key = AbsolutePathWithSlash $r.FileName + $HashTable[$key]=$r.CommitSha + } + return $HashTable +} + +function AttemptInvokeRestMethod($method, $url, $body, $contentTypes, $maxRetries) { + $Stoploop = $false + $retryCount = 0 + do { + try { + $result = Invoke-RestMethod -Uri $url -Method $method -Headers $header -Body $body -ContentType $contentTypes + $Stoploop = $true + } + catch { + if ($retryCount -gt $maxRetries) { + Write-Host "[Error] API call failed after $retryCount retries: $_" + $Stoploop = $true + } + else { + Write-Host "[Warning] API call failed: $_.`n Conducting retry #$retryCount." + Start-Sleep -Seconds 5 + $retryCount = $retryCount + 1 + } + } + } + While ($Stoploop -eq $false) + return $result +} + +function AttemptDeployMetadata($deploymentName, $resourceGroupName, $templateObject, $templateType, $paramFileType, $containsWorkspaceParam) { + $deploymentInfo = $null + try { + $deploymentInfo = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Ignore + } + catch { + Write-Host "[Warning] Unable to fetch deployment info for $deploymentName, no metadata was created for the resources in the file. Error: $_" + return + } + $deploymentInfo | Where-Object { $_.TargetResource -ne "" } | ForEach-Object { + $resource = $_.TargetResource + $sentinelContentKinds = GetContentKinds $resource + if ($sentinelContentKinds.Count -gt 0) { + $contentKind = ToContentKind $sentinelContentKinds $resource $templateObject + $contentId = $resource.Split("/")[-1] + $metadataCustomVersion = GetMetadataCustomVersion $templateType $paramFileType $containsWorkspaceParam + + $isSuccess = $false + $currentAttempt = 0 + + While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) + { + $currentAttempt ++ + Try + { + New-AzResourceGroupDeployment -Name "md-$deploymentName" -ResourceGroupName $ResourceGroupName -TemplateFile $metadataFilePath ` + -parentResourceId $resource ` + -kind $contentKind ` + -contentId $contentId ` + -sourceControlId $sourceControlId ` + -workspace $workspaceName ` + -customVersion $metadataCustomVersion ` + -ErrorAction Stop | Out-Host + Write-Host "[Info] Created metadata for $contentKind with parent resource id $resource" + $isSuccess = $true + } + Catch [Exception] + { + $err = $_ + if (-not (IsRetryable "md-$deploymentName")) + { + Write-Host "[Warning] Failed to deploy metadata for $contentKind with parent resource id $resource with error: $err" + break + } + else + { + if ($currentAttempt -le $MaxRetries) + { + Write-Host "[Warning] Failed to deploy metadata for $contentKind with error: $err. Retrying in $secondsBetweenAttempts seconds..." + Start-Sleep -Seconds $secondsBetweenAttempts + } + else + { + Write-Host "[Warning] Failed to deploy metadata for $contentKind after $currentAttempt attempts with error: $err" + } + } + } + } + } + } +} + +function GetMetadataCustomVersion($templateType, $paramFileType, $containsWorkspaceParam){ + $customVersion = $templateType + "-" + $paramFileType + if($containsWorkspaceParam){ + $customVersion += "-WorkspaceParam" + } + if($smartDeployment -eq "true"){ + $customVersion += "-SmartTracking" + } + return $customVersion +} + +function GetContentKinds($resource) { + return $sentinelResourcePatterns.Keys | Where-Object { $resource -match $sentinelResourcePatterns[$_] } +} + +function ToContentKind($contentKinds, $resource, $templateObject) { + if ($contentKinds.Count -eq 1) { + return $contentKinds + } + if ($null -ne $resource -and $resource.Contains('savedSearches')) { + if ($templateObject.resources.properties.Category -eq "Hunting Queries") { + return "HuntingQuery" + } + return "Parser" + } + return $null +} + +function IsValidTemplate($path, $templateObject, $parameterFile) { + Try { + if (DoesContainWorkspaceParam $templateObject) { + if ($parameterFile) { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile -workspace $WorkspaceName + } + else { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName + } + } + else { + if ($parameterFile) { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile + } else { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path + } + } + + return $true + } + Catch { + Write-Host "[Warning] The file $path is not valid: $_" + return $false + } +} + +function IsRetryable($deploymentName) { + $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" + Try { + $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop + return $retryableStatusCodes -contains $deploymentResult.StatusCode + } + Catch { + return $false + } +} + +function IsValidResourceType($template) { + try { + $isAllowedResources = $true + $template.resources | ForEach-Object { + $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources + } + } + catch { + Write-Host "[Error] Failed to check valid resource type." + $isAllowedResources = $false + } + return $isAllowedResources +} + +function DoesContainWorkspaceParam($templateObject) { + $templateObject.parameters.PSobject.Properties.Name -contains "workspace" +} + +function AttemptDeployment($path, $parameterFile, $deploymentName, $templateObject, $templateType) { + Write-Host "[Info] Deploying $path with deployment name $deploymentName" + + $isValid = IsValidTemplate $path $templateObject $parameterFile + if (-not $isValid) { + Write-Host "[Error] Not deploying $path since the template is not valid" + return $false + } + $isSuccess = $false + $currentAttempt = 0 + While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) + { + $currentAttempt ++ + Try + { + Write-Host "[Info] Deploy $path with parameter file: [$parameterFile]" + $paramFileType = if(!$parameterFile) {"NoParam"} elseif($parameterFile -like "*.bicepparam") {"BicepParam"} else {"JsonParam"} + $containsWorkspaceParam = DoesContainWorkspaceParam $templateObject + if ($containsWorkspaceParam) + { + if ($parameterFile) { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host + } + else + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host + } + } + else + { + if ($parameterFile) { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host + } + else + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host + } + } + AttemptDeployMetadata $deploymentName $ResourceGroupName $templateObject $templateType $paramFileType $containsWorkspaceParam + + $isSuccess = $true + } + Catch [Exception] + { + $err = $_ + if (-not (IsRetryable $deploymentName)) + { + Write-Host "[Warning] Failed to deploy $path with error: $err" + break + } + else + { + if ($currentAttempt -le $MaxRetries) + { + Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." + Start-Sleep -Seconds $secondsBetweenAttempts + } + else + { + Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" + } + } + } + } + return $isSuccess +} + +function GenerateDeploymentName() { + $randomId = [guid]::NewGuid() + return "Sentinel_Deployment_$randomId" +} + +#Load deployment configuration +function LoadDeploymentConfig() { + Write-Host "[Info] load the deployment configuration from [$configPath]" + $global:parameterFileMapping = @{} + $global:prioritizedContentFiles = @() + $global:excludeContentFiles = @() + try { + if (Test-Path $configPath) { + $deployment_config = Get-Content $configPath | Out-String | ConvertFrom-Json + $parameterFileMappings = @{} + if ($deployment_config.parameterfilemappings) { + $deployment_config.parameterfilemappings.psobject.properties | ForEach { $parameterFileMappings[$_.Name] = $_.Value } + } + $key = ($parameterFileMappings.Keys | ? { $_ -eq $workspaceId }) + if ($null -ne $key) { + $parameterFileMappings[$key].psobject.properties | ForEach { $global:parameterFileMapping[$_.Name] = $_.Value } + } + if ($deployment_config.prioritizedcontentfiles) { + $global:prioritizedContentFiles = $deployment_config.prioritizedcontentfiles + } + $excludeList = $global:parameterFileMapping.Values + $global:prioritizedcontentfiles + if ($deployment_config.excludecontentfiles) { + $excludeList = $excludeList + $deployment_config.excludecontentfiles + } + $global:excludeContentFiles = $excludeList | Where-Object { Test-Path (AbsolutePathWithSlash $_) } + } + } + catch { + Write-Host "[Warning] An error occurred while trying to load deployment configuration." + Write-Host "Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function filterContentFile($fullPath) { + $temp = RelativePathWithBackslash $fullPath + return $global:excludeContentFiles | Where-Object {$temp.StartsWith($_, 'CurrentCultureIgnoreCase')} +} + +function RelativePathWithBackslash($absolutePath) { + return $absolutePath.Replace($rootDirectory + "\", "").Replace("\", "/") +} + +function AbsolutePathWithSlash($relativePath) { + return Join-Path -Path $rootDirectory -ChildPath $relativePath +} + +#resolve parameter file name, return $null if there is none. +function GetParameterFile($path) { + if ($path.Length -eq 0) { + return $null + } + + $index = RelativePathWithBackslash $path + $key = ($global:parameterFileMapping.Keys | Where-Object { $_ -eq $index }) + if ($key) { + $mappedParameterFile = AbsolutePathWithSlash $global:parameterFileMapping[$key] + if (Test-Path $mappedParameterFile) { + return $mappedParameterFile + } + } + + $extension = [System.IO.Path]::GetExtension($path) + if ($extension -ne ".json" -and $extension -ne ".bicep") { + return $null + } + + $parameterFilePrefix = $path.Substring(0, $path.Length - $extension.Length) + + # Check for workspace-specific parameter file + if ($extension -eq ".bicep") { + $workspaceParameterFile = $parameterFilePrefix + "-$WorkspaceId.bicepparam" + if (Test-Path $workspaceParameterFile) { + return $workspaceParameterFile + } + } + + $workspaceParameterFile = $parameterFilePrefix + ".parameters-$WorkspaceId.json" + if (Test-Path $workspaceParameterFile) { + return $workspaceParameterFile + } + + # Check for parameter file + if ($extension -eq ".bicep") { + $defaultParameterFile = $parameterFilePrefix + ".bicepparam" + Write-Host "Default parameter file: $defaultParameterFile" + if (Test-Path $defaultParameterFile) { + return $defaultParameterFile + } + } + + $defaultParameterFile = $parameterFilePrefix + ".parameters.json" + Write-Host "Default parameter file: $defaultParameterFile" + if (Test-Path $defaultParameterFile) { + return $defaultParameterFile + } + + return $null +} + +function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { + Write-Host "Starting Deployment for Files in path: $Directory" + if (Test-Path -Path $Directory) + { + $totalFiles = 0; + $totalFailed = 0; + $iterationList = @() + $global:prioritizedContentFiles | ForEach-Object { $iterationList += (AbsolutePathWithSlash $_) } + Get-ChildItem -Path $Directory -Recurse -Include *.bicep, *.json -exclude *metadata.json, *.parameters*.json, *.bicepparam, bicepconfig.json | + Where-Object { $null -eq ( filterContentFile $_.FullName ) } | + Select-Object -Property FullName | + ForEach-Object { $iterationList += $_.FullName } + $iterationList | ForEach-Object { + $path = $_ + Write-Host "[Info] Try to deploy $path" + if (-not (Test-Path $path)) { + Write-Host "[Warning] Skipping deployment for $path. The file doesn't exist." + return + } + + if ($path -like "*.bicep") { + $templateType = "Bicep" + $templateObject = bicep build $path --stdout | Out-String | ConvertFrom-Json + } else { + $templateType = "ARM" + $templateObject = Get-Content $path | Out-String | ConvertFrom-Json + } + + if (-not (IsValidResourceType $templateObject)) + { + Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." + return + } + $parameterFile = GetParameterFile $path + $result = SmartDeployment $fullDeploymentFlag $remoteShaTable $path $parameterFile $templateObject $templateType + if ($result.isSuccess -eq $false) { + $totalFailed++ + } + if (-not $result.skip) { + $totalFiles++ + } + if ($result.isSuccess -or $result.skip) { + $global:updatedCsvTable[$path] = $remoteShaTable[$path] + if ($parameterFile) { + $global:updatedCsvTable[$parameterFile] = $remoteShaTable[$parameterFile] + } + } + } + PushCsvToRepo + if ($totalFiles -gt 0 -and $totalFailed -gt 0) + { + $err = "$totalFailed of $totalFiles deployments failed." + Throw $err + } + } + else + { + Write-Output "[Warning] $Directory not found. nothing to deploy" + } +} + +function SmartDeployment($fullDeploymentFlag, $remoteShaTable, $path, $parameterFile, $templateObject, $templateType) { + try { + $skip = $false + $isSuccess = $null + if (!$fullDeploymentFlag) { + $existingSha = $global:localCsvTablefinal[$path] + $remoteSha = $remoteShaTable[$path] + $skip = (($existingSha) -and ($existingSha -eq $remoteSha)) + if ($skip -and $parameterFile) { + $existingShaForParameterFile = $global:localCsvTablefinal[$parameterFile] + $remoteShaForParameterFile = $remoteShaTable[$parameterFile] + $skip = (($existingShaForParameterFile) -and ($existingShaForParameterFile -eq $remoteShaForParameterFile)) + } + } + if (!$skip) { + $deploymentName = GenerateDeploymentName + $isSuccess = AttemptDeployment $path $parameterFile $deploymentName $templateObject $templateType + } + return @{ + skip = $skip + isSuccess = $isSuccess + } + } + catch { + Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function TryGetCsvFile { + if (Test-Path $csvPath) { + $global:localCsvTablefinal = ReadCsvToTable + Remove-Item -Path $csvPath + git add $csvPath + git commit -m "Removed tracking file and moved to new sentinel created branch" + git push origin $branchName + } + + $relativeCsvPath = RelativePathWithBackslash $csvPath + $resourceBranchExists = git ls-remote --heads "https://github.com/$githubRepository" $newResourceBranch | wc -l + + if ($resourceBranchExists -eq 1) { + git fetch > $null + git checkout $newResourceBranch + + if (Test-Path $relativeCsvPath) { + $global:localCsvTablefinal = ReadCsvToTable + } + git checkout $branchName + } +} + +function main() { + git config --global user.email "donotreply@microsoft.com" + git config --global user.name "Sentinel" + + TryGetCsvFile + LoadDeploymentConfig + $tree = GetGithubTree + $remoteShaTable = GetCommitShaTable $tree + + $existingConfigSha = $global:localCsvTablefinal[$configPath] + $remoteConfigSha = $remoteShaTable[$configPath] + $modifiedConfig = ($existingConfigSha -xor $remoteConfigSha) -or ($existingConfigSha -and $remoteConfigSha -and ($existingConfigSha -ne $remoteConfigSha)) + + if ($remoteConfigSha) { + $global:updatedCsvTable[$configPath] = $remoteConfigSha + } + + $fullDeploymentFlag = $modifiedConfig -or ($smartDeployment -eq "false") + Deployment $fullDeploymentFlag $remoteShaTable $tree +} + +main \ No newline at end of file diff --git a/.github/workflows/sentinel-deploy-70b27887-856e-4f03-a7f0-2cec8af84042.yml b/.github/workflows/sentinel-deploy-70b27887-856e-4f03-a7f0-2cec8af84042.yml new file mode 100644 index 000000000..215008ad2 --- /dev/null +++ b/.github/workflows/sentinel-deploy-70b27887-856e-4f03-a7f0-2cec8af84042.yml @@ -0,0 +1,128 @@ +name: Deploy Content to testrolemtp [70b27887-856e-4f03-a7f0-2cec8af84042] +# Note: This workflow will deploy everything in the root directory. +# To deploy content only from a specific path (for example SentinelContent): +# 1. Add the target path to the "paths" property like such +# paths: +# - 'SentinelContent/**' +# - '!.github/workflows/**' +# - '.github/workflows/sentinel-deploy-70b27887-856e-4f03-a7f0-2cec8af84042.yml' +# 2. Append the path to the directory environment variable below +# directory: '${{ github.workspace }}/SentinelContent' + +on: + push: + branches: [ patch-1 ] + paths: + - '**' + - '!.github/workflows/**' # this filter prevents other workflow changes from triggering this workflow + - '.github/workflows/sentinel-deploy-70b27887-856e-4f03-a7f0-2cec8af84042.yml' + +jobs: + deploy-content: + runs-on: windows-latest + env: + resourceGroupName: 'loganalyticstest' + workspaceName: 'testrolemtp' + workspaceId: 'f2aed523-c473-4a2f-a94a-010ab6448743' + directory: '${{ github.workspace }}' + cloudEnv: 'AzurePPE' + contentTypes: 'CustomDetection' + branch: 'patch-1' + sourceControlId: '70b27887-856e-4f03-a7f0-2cec8af84042' + rootDirectory: '${{ github.workspace }}' + githubAuthToken: ${{ secrets.GITHUB_TOKEN }} + smartDeployment: 'true' + permissions: + contents: write + id-token: write # Require write permission to Fetch an OIDC token. + + steps: + - name: Login to Azure (Attempt 1) + continue-on-error: true + id: login1 + uses: azure/powershell@v2 + with: + azPSVersion: 'latest' + inlineScript: | + Add-AzEnvironment ` + -Name AzurePPE ` + -ActiveDirectoryEndpoint https://login.windows-ppe.net ` + -ResourceManagerEndpoint https://management.azure.com/ ` + -ActiveDirectoryServiceEndpointResourceId https://management.core.windows.net ` + -GraphEndpoint https://graph.ppe.windows.net/ | out-null; + $oidcTokenParams = @{ + Uri = $env:ACTIONS_ID_TOKEN_REQUEST_URL + Body = @{ + audience = 'api://AzureADTokenExchange' + } + Authentication = 'Bearer' + Token = $env:ACTIONS_ID_TOKEN_REQUEST_TOKEN | ConvertTo-SecureString -AsPlainText + } + $IdToken = (Invoke-RestMethod @oidcTokenParams).value + Connect-AzAccount -ServicePrincipal -Tenant ${{ secrets.AZURE_SENTINEL_TENANTID_70b27887856e4f03a7f02cec8af84042 }} -Subscription ${{ secrets.AZURE_SENTINEL_SUBSCRIPTIONID_70b27887856e4f03a7f02cec8af84042 }} -ApplicationId ${{ secrets.AZURE_SENTINEL_CLIENTID_70b27887856e4f03a7f02cec8af84042 }} -Environment AzurePPE -FederatedToken $IdToken | out-null; + + - name: Wait 30 seconds if login attempt 1 failed + if: ${{ steps.login1.outcome=='failure' }} + run: powershell Start-Sleep -s 30 + + - name: Login to Azure (Attempt 2) + continue-on-error: true + id: login2 + uses: azure/powershell@v2 + with: + azPSVersion: 'latest' + inlineScript: | + Add-AzEnvironment ` + -Name AzurePPE ` + -ActiveDirectoryEndpoint https://login.windows-ppe.net ` + -ResourceManagerEndpoint https://management.azure.com/ ` + -ActiveDirectoryServiceEndpointResourceId https://management.core.windows.net ` + -GraphEndpoint https://graph.ppe.windows.net/ | out-null; + $oidcTokenParams = @{ + Uri = $env:ACTIONS_ID_TOKEN_REQUEST_URL + Body = @{ + audience = 'api://AzureADTokenExchange' + } + Authentication = 'Bearer' + Token = $env:ACTIONS_ID_TOKEN_REQUEST_TOKEN | ConvertTo-SecureString -AsPlainText + } + $IdToken = (Invoke-RestMethod @oidcTokenParams).value + Connect-AzAccount -ServicePrincipal -Tenant ${{ secrets.AZURE_SENTINEL_TENANTID_70b27887856e4f03a7f02cec8af84042 }} -Subscription ${{ secrets.AZURE_SENTINEL_SUBSCRIPTIONID_70b27887856e4f03a7f02cec8af84042 }} -ApplicationId ${{ secrets.AZURE_SENTINEL_CLIENTID_70b27887856e4f03a7f02cec8af84042 }} -Environment AzurePPE -FederatedToken $IdToken | out-null; + + - name: Wait 30 seconds if login attempt 2 failed + if: ${{ steps.login2.outcome=='failure' }} + run: powershell Start-Sleep -s 30 + + - name: Login to Azure (Attempt 3) + continue-on-error: false + id: login3 + uses: azure/powershell@v2 + with: + azPSVersion: 'latest' + inlineScript: | + Add-AzEnvironment ` + -Name AzurePPE ` + -ActiveDirectoryEndpoint https://login.windows-ppe.net ` + -ResourceManagerEndpoint https://management.azure.com/ ` + -ActiveDirectoryServiceEndpointResourceId https://management.core.windows.net ` + -GraphEndpoint https://graph.ppe.windows.net/ | out-null; + $oidcTokenParams = @{ + Uri = $env:ACTIONS_ID_TOKEN_REQUEST_URL + Body = @{ + audience = 'api://AzureADTokenExchange' + } + Authentication = 'Bearer' + Token = $env:ACTIONS_ID_TOKEN_REQUEST_TOKEN | ConvertTo-SecureString -AsPlainText + } + $IdToken = (Invoke-RestMethod @oidcTokenParams).value + Connect-AzAccount -ServicePrincipal -Tenant ${{ secrets.AZURE_SENTINEL_TENANTID_70b27887856e4f03a7f02cec8af84042 }} -Subscription ${{ secrets.AZURE_SENTINEL_SUBSCRIPTIONID_70b27887856e4f03a7f02cec8af84042 }} -ApplicationId ${{ secrets.AZURE_SENTINEL_CLIENTID_70b27887856e4f03a7f02cec8af84042 }} -Environment AzurePPE -FederatedToken $IdToken | out-null; + + - name: Checkout + uses: actions/checkout@v3 + + - name: Deploy Content to Microsoft Sentinel + uses: azure/powershell@v2 + with: + azPSVersion: 'latest' + inlineScript: | + ${{ github.workspace }}//.github/workflows/azure-sentinel-deploy-70b27887-856e-4f03-a7f0-2cec8af84042.ps1 \ No newline at end of file diff --git a/.github/workflows/sentinel-deploy-e7383813-bbb9-4fd8-b4dc-743beac9ee13.yml b/.github/workflows/sentinel-deploy-e7383813-bbb9-4fd8-b4dc-743beac9ee13.yml new file mode 100644 index 000000000..9c3ba1dfd --- /dev/null +++ b/.github/workflows/sentinel-deploy-e7383813-bbb9-4fd8-b4dc-743beac9ee13.yml @@ -0,0 +1,94 @@ +name: Deploy Content to tal-test [e7383813-bbb9-4fd8-b4dc-743beac9ee13] +# Note: This workflow will deploy everything in the root directory. +# To deploy content only from a specific path (for example SentinelContent): +# 1. Add the target path to the "paths" property like such +# paths: +# - 'SentinelContent/**' +# - '!.github/workflows/**' +# - '.github/workflows/sentinel-deploy-e7383813-bbb9-4fd8-b4dc-743beac9ee13.yml' +# 2. Append the path to the directory environment variable below +# directory: '${{ github.workspace }}/SentinelContent' + +on: + push: + branches: [ patch-1 ] + paths: + - '**' + - '!.github/workflows/**' # this filter prevents other workflow changes from triggering this workflow + - '.github/workflows/sentinel-deploy-e7383813-bbb9-4fd8-b4dc-743beac9ee13.yml' + +jobs: + deploy-content: + runs-on: windows-latest + env: + resourceGroupName: 'tal-test-rg' + workspaceName: 'tal-test' + workspaceId: '671511bf-526c-449e-956d-ce98e9d320c3' + directory: '${{ github.workspace }}' + cloudEnv: 'AzureCloud' + contentTypes: 'AnalyticsRule,AutomationRule,HuntingQuery,Parser,Playbook,Workbook' + branch: 'patch-1' + sourceControlId: 'e7383813-bbb9-4fd8-b4dc-743beac9ee13' + rootDirectory: '${{ github.workspace }}' + githubAuthToken: ${{ secrets.GITHUB_TOKEN }} + smartDeployment: 'true' + permissions: + contents: write + id-token: write # Require write permission to Fetch an OIDC token. + + steps: + - name: Login to Azure (Attempt 1) + continue-on-error: true + id: login1 + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_SENTINEL_CLIENTID_e7383813bbb94fd8b4dc743beac9ee13 }} + tenant-id: ${{ secrets.AZURE_SENTINEL_TENANTID_e7383813bbb94fd8b4dc743beac9ee13 }} + subscription-id: ${{ secrets.AZURE_SENTINEL_SUBSCRIPTIONID_e7383813bbb94fd8b4dc743beac9ee13 }} + environment: 'AzureCloud' + audience: api://AzureADTokenExchange + enable-AzPSSession: true + + - name: Wait 30 seconds if login attempt 1 failed + if: ${{ steps.login1.outcome=='failure' }} + run: powershell Start-Sleep -s 30 + + - name: Login to Azure (Attempt 2) + continue-on-error: true + id: login2 + uses: azure/login@v2 + if: ${{ steps.login1.outcome=='failure' }} + with: + client-id: ${{ secrets.AZURE_SENTINEL_CLIENTID_e7383813bbb94fd8b4dc743beac9ee13 }} + tenant-id: ${{ secrets.AZURE_SENTINEL_TENANTID_e7383813bbb94fd8b4dc743beac9ee13 }} + subscription-id: ${{ secrets.AZURE_SENTINEL_SUBSCRIPTIONID_e7383813bbb94fd8b4dc743beac9ee13 }} + environment: 'AzureCloud' + audience: api://AzureADTokenExchange + enable-AzPSSession: true + + - name: Wait 30 seconds if login attempt 2 failed + if: ${{ steps.login2.outcome=='failure' }} + run: powershell Start-Sleep -s 30 + + - name: Login to Azure (Attempt 3) + continue-on-error: false + id: login3 + uses: azure/login@v2 + if: ${{ steps.login2.outcome=='failure' }} + with: + client-id: ${{ secrets.AZURE_SENTINEL_CLIENTID_e7383813bbb94fd8b4dc743beac9ee13 }} + tenant-id: ${{ secrets.AZURE_SENTINEL_TENANTID_e7383813bbb94fd8b4dc743beac9ee13 }} + subscription-id: ${{ secrets.AZURE_SENTINEL_SUBSCRIPTIONID_e7383813bbb94fd8b4dc743beac9ee13 }} + environment: 'AzureCloud' + audience: api://AzureADTokenExchange + enable-AzPSSession: true + + - name: Checkout + uses: actions/checkout@v3 + + - name: Deploy Content to Microsoft Sentinel + uses: azure/powershell@v2 + with: + azPSVersion: 'latest' + inlineScript: | + ${{ github.workspace }}//.github/workflows/azure-sentinel-deploy-e7383813-bbb9-4fd8-b4dc-743beac9ee13.ps1 \ No newline at end of file diff --git a/README.md b/README.md index d1c4fbab0..f8f9cd31d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repository provides samples of deployable Sentinel content as well as examp **Please note** that this repository contains sample content that is not intended to be used as or in the place of any real security content. The sole intention of this repository is to help demonstrate the capabilities of Microsoft Sentinel Repositories. # The Sample Content Folders -You can find a variety of supported content to use in your test deployments in the respective content folders of this repository. In addition, you can utilize the JSON or YAML to ARM scripts we've included in some folders (e.g. Detections, Hunting, and Workbooks) to convert your content files to the supported ARM format for repositories deployment. Please note that these scripts were used to convert some of the content in the [Azure Sentinel Community Repository](https://github.com/Azure/Azure-Sentinel) but have not been tested on all variations of content, please use with care. +You can find a variety of supported content to use in your test deployments in the respective content folders of this repository. In addition, you can utilize the JSON or YAML to ARM scripts we've included in some folders (e.g. Detections, Hunting and Workbooks) to convert your content files to the supported ARM format for repositories deployment. Please note that these scripts were used to convert some of the content in the [Azure Sentinel Community Repository](https://github.com/Azure/Azure-Sentinel) but have not been tested on all variations of content, please use with care. # Scaling your CICD pipeline ## Sentinel Deployment Configuration