durable cleanup

Author: JohnDuprey

CIPPTimers.json

@@ -1,4 +1,12 @@
 [
+  {
+    "Command": "Start-DurableCleanup",
+    "Description": "Timer function to cleanup durable functions",
+    "Cron": "0 */15 * * * *",
+    "Priority": 0,
+    "RunOnProcessor": true,
+    "IsSystem": true
+  },
   {
     "Command": "Start-UserTasksOrchestrator",
     "Description": "Orchestrator to process user scheduled tasks",

Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-DurableCleanup.ps1

@@ -0,0 +1,61 @@
+function Start-DurableCleanup {
+    <#
+    .SYNOPSIS
+    Start the durable cleanup process.
+
+    .DESCRIPTION
+    Look for orchestrators running for more than the specified time and terminate them. Also, clear any queues that have items for that function app.
+
+    .PARAMETER MaxDuration
+    The maximum duration an orchestrator can run before being terminated.
+
+    .FUNCTIONALITY
+    Internal
+    #>
+
+    [CmdletBinding(SupportsShouldProcess = $true)]
+    param(
+        [int]$MaxDuration = 3600
+    )
+    $WarningPreference = 'SilentlyContinue'
+    $StorageContext = New-AzStorageContext -ConnectionString $env:AzureWebJobsStorage
+    $TargetTime = (Get-Date).ToUniversalTime().AddSeconds(-$MaxDuration)
+    $Context = New-AzDataTableContext -ConnectionString $env:AzureWebJobsStorage
+    $InstancesTables = Get-AzDataTable -Context $Context | Where-Object { $_ -match 'Instances' }
+
+    $CleanupCount = 0
+    $QueueCount = 0
+    foreach ($Table in $InstancesTables) {
+        $Table = Get-CippTable -TableName $Table
+        $ClearQueues = $false
+        $FunctionName = $Table.TableName -replace 'Instances', ''
+        $Orchestrators = Get-CIPPAzDataTableEntity @Table -Filter "RuntimeStatus eq 'Running'" | Select-Object * -ExcludeProperty Input
+        $Orchestrators | Where-Object { $_.CreatedTime.DateTime -lt $TargetTime } | ForEach-Object {
+            $CreatedTime = [DateTime]::SpecifyKind($_.CreatedTime.DateTime, [DateTimeKind]::Utc)
+            $TimeSpan = New-TimeSpan -Start $CreatedTime -End (Get-Date).ToUniversalTime()
+            $RunningDuration = [math]::Round($TimeSpan.TotalMinutes, 2)
+            Write-Information "Orchestrator: $($_.PartitionKey), created: $CreatedTime, running for: $RunningDuration minutes"
+            $ClearQueues = $true
+            $_.RuntimeStatus = 'Failed'
+            if ($PSCmdlet.ShouldProcess($_.PartitionKey, 'Terminate Orchestrator')) {
+                $Orchestrator = Get-CIPPAzDataTableEntity @Table -PartitionKey $_.PartitionKey -RowKey $_.RowKey
+                $Orchestrator.RuntimeStatus = 'Failed'
+                Update-AzDataTableEntity @Table -Entity $Orchestrator
+                $CleanupCount++
+            }
+        }
+
+        if ($ClearQueues) {
+            $Queues = Get-AzStorageQueue -Context $StorageContext -Name ('{0}*' -f $FunctionName) | Select-Object -Property Name, ApproximateMessageCount, QueueClient
+            $RunningQueues = $Queues | Where-Object { $_.ApproximateMessageCount -gt 0 }
+            foreach ($Queue in $RunningQueues) {
+                Write-Information "- Removing queue: $($Queue.Name), message count: $($Queue.ApproximateMessageCount)"
+                if ($PSCmdlet.ShouldProcess($Queue.Name, 'Clear Queue')) {
+                    $Queue.QueueClient.ClearMessagesAsync() | Out-Null
+                }
+                $QueueCount++
+            }
+        }
+    }
+    Write-Information "Cleanup complete. $CleanupCount orchestrators were terminated. $QueueCount queues were cleared."
+}