Add ARM throttling dashboards for FXCI subscriptions

Author: jwmossmoz

terraform/azure_fxci/arm-throttling-dashboard.tf

@@ -0,0 +1,11 @@
+module "arm_subscription_throttling_dashboard" {
+  source = "../azure_modules/arm_subscription_throttling_dashboard"
+
+  resource_group_id         = azurerm_resource_group.rg-central-us-runbooks.id
+  location                  = azurerm_resource_group.rg-central-us-runbooks.location
+  subscription_id           = "108d46d5-fe9b-4850-9a7d-8c914aa6c1f0"
+  subscription_display_name = "FXCI Azure DevTest Subscription"
+  dashboard_name            = "arm-throttle-fxci"
+  dashboard_display_name    = "FXCI ARM Throttling"
+  tags                      = local.common_tags
+}

terraform/azure_modules/arm_subscription_throttling_dashboard/main.tf

@@ -0,0 +1,159 @@
+terraform {
+  required_providers {
+    azapi = {
+      source = "azure/azapi"
+    }
+  }
+}
+
+locals {
+  subscription_resource_id = "/subscriptions/${var.subscription_id}"
+
+  overview_markdown = <<-MARKDOWN
+  ## ${var.dashboard_display_name}
+  This dashboard is the shared entry point for Azure Resource Manager subscription throttling in ${var.subscription_display_name}.
+
+  - Subscription resource: `${local.subscription_resource_id}`
+  - Start with the `Traffic` metric and add `StatusCode = 429`.
+  - High-signal filters for the current FXCI issue: `Namespace = MICROSOFT.RESOURCES`, `OperationType = write`.
+  - Click a chart to open Metrics explorer and split or filter by `RequestRegion`, `Namespace`, or `OperationType`.
+  MARKDOWN
+
+  drilldown_markdown = <<-MARKDOWN
+  ### How to confirm subscription bucket pressure
+  1. Re-run a failing `az` command with `--debug`.
+  2. Capture the `429` response and inspect `x-ms-ratelimit-remaining-subscription-reads` or `x-ms-ratelimit-remaining-subscription-writes`.
+  3. If those headers are near `0`, the general ARM subscription bucket is being exhausted.
+
+  Docs:
+  - [Monitor Azure Resource Manager](https://learn.microsoft.com/azure/azure-resource-manager/management/monitor-resource-manager)
+  - [Understand how Azure Resource Manager throttles requests](https://learn.microsoft.com/azure/azure-resource-manager/management/request-limits-and-throttling)
+  MARKDOWN
+}
+
+resource "azapi_resource" "this" {
+  type                      = "Microsoft.Portal/dashboards@2020-09-01-preview"
+  name                      = var.dashboard_name
+  parent_id                 = var.resource_group_id
+  location                  = var.location
+  schema_validation_enabled = false
+
+  tags = merge(
+    var.tags,
+    {
+      Name           = var.dashboard_name
+      "hidden-title" = var.dashboard_display_name
+    }
+  )
+
+  body = {
+    properties = {
+      lenses = [
+        {
+          order = 0
+          parts = [
+            {
+              position = {
+                x       = 0
+                y       = 0
+                rowSpan = 3
+                colSpan = 11
+              }
+              metadata = {
+                inputs = []
+                type   = "Extension/HubsExtension/PartType/MarkdownPart"
+                settings = {
+                  content = {
+                    settings = {
+                      content  = local.overview_markdown
+                      title    = var.dashboard_display_name
+                      subtitle = "Azure Resource Manager subscription throttling"
+                    }
+                  }
+                }
+              }
+            },
+            {
+              position = {
+                x       = 0
+                y       = 3
+                rowSpan = 4
+                colSpan = 6
+              }
+              metadata = {
+                inputs = [
+                  {
+                    name = "queryInputs"
+                    value = {
+                      timespan = {
+                        duration = var.metric_timespan
+                      }
+                      id        = local.subscription_resource_id
+                      chartType = 0
+                      metrics = [
+                        {
+                          name       = "Traffic"
+                          resourceId = local.subscription_resource_id
+                        }
+                      ]
+                    }
+                  }
+                ]
+                type = "Extension/Microsoft_Azure_Monitoring/PartType/MetricsChartPart"
+              }
+            },
+            {
+              position = {
+                x       = 6
+                y       = 3
+                rowSpan = 4
+                colSpan = 5
+              }
+              metadata = {
+                inputs = [
+                  {
+                    name = "queryInputs"
+                    value = {
+                      timespan = {
+                        duration = var.metric_timespan
+                      }
+                      id        = local.subscription_resource_id
+                      chartType = 0
+                      metrics = [
+                        {
+                          name       = "Latency"
+                          resourceId = local.subscription_resource_id
+                        }
+                      ]
+                    }
+                  }
+                ]
+                type = "Extension/Microsoft_Azure_Monitoring/PartType/MetricsChartPart"
+              }
+            },
+            {
+              position = {
+                x       = 0
+                y       = 7
+                rowSpan = 5
+                colSpan = 11
+              }
+              metadata = {
+                inputs = []
+                type   = "Extension/HubsExtension/PartType/MarkdownPart"
+                settings = {
+                  content = {
+                    settings = {
+                      content = local.drilldown_markdown
+                      title   = "Operational notes"
+                    }
+                  }
+                }
+              }
+            }
+          ]
+        }
+      ]
+    }
+  }
+}

terraform/azure_modules/arm_subscription_throttling_dashboard/outputs.tf

@@ -0,0 +1,4 @@
+output "dashboard_id" {
+  description = "Resource ID for the Azure Portal dashboard."
+  value       = azapi_resource.this.id
+}

terraform/azure_modules/arm_subscription_throttling_dashboard/variables.tf

@@ -0,0 +1,40 @@
+variable "dashboard_display_name" {
+  description = "Portal display name for the dashboard."
+  type        = string
+}
+
+variable "dashboard_name" {
+  description = "Azure resource name for the dashboard."
+  type        = string
+}
+
+variable "location" {
+  description = "Azure region where the dashboard resource will live."
+  type        = string
+}
+
+variable "metric_timespan" {
+  description = "Default timespan for dashboard metrics."
+  type        = string
+  default     = "P7D"
+}
+
+variable "resource_group_id" {
+  description = "Resource group resource ID used as the dashboard parent."
+  type        = string
+}
+
+variable "subscription_display_name" {
+  description = "Human-readable subscription name shown in dashboard markdown."
+  type        = string
+}
+
+variable "subscription_id" {
+  description = "Subscription GUID for the monitored Azure subscription."
+  type        = string
+}
+
+variable "tags" {
+  description = "Tags applied to the dashboard resource."
+  type        = map(string)
+}

terraform/azure_trusted_fxci/arm-throttling-dashboard.tf

@@ -0,0 +1,11 @@
+module "arm_subscription_throttling_dashboard" {
+  source = "../azure_modules/arm_subscription_throttling_dashboard"
+
+  resource_group_id         = azurerm_resource_group.rg-central-us-trusted-runbooks.id
+  location                  = azurerm_resource_group.rg-central-us-trusted-runbooks.location
+  subscription_id           = "a30e97ab-734a-4f3b-a0e4-c51c0bff0701"
+  subscription_display_name = "Trusted FXCI Azure DevTest Subscription"
+  dashboard_name            = "arm-throttle-tfxci"
+  dashboard_display_name    = "Trusted FXCI ARM Throttling"
+  tags                      = local.common_tags
+}