Merge branch 'KelvinTegelaar:master' into master

Author: luke255

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+  "recommendations": [
+    "editorconfig.editorconfig"
+  ]
+}

Cache_SAMSetup/PermissionsTranslator.json

@@ -1004,8 +1004,15 @@
     "description": "Allows the app to create, read, update, and delete events of all calendars without a signed-in user.",
     "displayName": "Read and write calendars in all mailboxes",
     "id": "ef54d2bf-783f-4e0f-bca1-3210c0444d99",
-    "origin": "Application",
-    "value": "Calendars.ReadWrite"
+    "origin": "Application (Office 365 Exchange Online)",
+    "value": "Calendars.ReadWrite.All"
+  },
+  {
+    "description": "Allows the app to create, read, update, and delete user's mailbox settings without a signed-in user. Does not include permission to send mail.",
+    "displayName": "Read and write all user mailbox settings",
+    "id": "f9156939-25cd-4ba8-abfe-7fabcf003749",
+    "origin": "Application (Office 365 Exchange Online)",
+    "value": "Mailbox.Settings.ReadWrite"
   },
   {
     "description": "Allows the app to read your organization's user flows, without a signed-in user.",
@@ -5286,6 +5293,24 @@
     "userConsentDisplayName": "Read Threat and Vulnerability Management vulnerability information",
     "value": "Exchange.Manage"
   },
+  {
+    "description": "Allows the app to create, read, update and delete events in all calendars in the organization user has permissions to access. This includes delegate and shared calendars",
+    "displayName": "Read and write user and shared calendars",
+    "id": "bbd1ca91-75e0-4814-ad94-9c5dbbae3415",
+    "Origin": "Delegated (Office 365 Exchange Online)",
+    "userConsentDescription": "Allows the app to read, update, create and delete events in all calendars in your organization you have permissions to access. This includes delegate and shared calendars",
+    "userConsentDisplayName": "Read and write to your and shared calendars",
+    "value": "Calendars.ReadWrite.All"
+  },
+  {
+    "description": "Allows the app to create, read, update, and delete user's mailbox settings. Does not include permission to send mail.",
+    "displayName": "Read and write user mailbox settings",
+    "id": "2e83d72d-8895-4b66-9eea-abb43449ab8b",
+    "Origin": "Delegated (Office 365 Exchange Online)",
+    "userConsentDescription": "Allows the app to read, update, create, and delete your mailbox settings.",
+    "userConsentDisplayName": "Read and write to your mailbox settings",
+    "value": "MailboxSettings.ReadWrite"
+  },
   {
     "description": "Allows the app to have full control of all site collections on behalf of the signed-in user.",
     "displayName": "Manage Sharepoint Online",
@@ -5312,5 +5337,14 @@
     "userConsentDescription": "Access Microsoft Teams and Skype for Business data as the signed in user",
     "userConsentDisplayName": "Access Microsoft Teams and Skype for Business data based on the user's role membership",
     "value": "user_impersonation"
+  },
+  {
+    "description": "Read and write all on-premises directory synchronization information",
+    "displayName": "Read and write all on-premises directory synchronization information",
+    "id": "c2d95988-7604-4ba1-aaed-38a5f82a51c7",
+    "Origin": "Delegated",
+    "userConsentDescription": "Access Microsoft Teams and Skype for Business data as the signed in user",
+    "userConsentDisplayName": "Access Microsoft Teams and Skype for Business data based on the user's role membership",
+    "value": "OnPremDirectorySynchronization.ReadWrite.All"
   }
 ]

Cache_SAMSetup/SAMManifest.json

@@ -10,7 +10,7 @@ Write-Host "Working on $UserName"
 try {
   $startDate = (Get-Date).AddDays(-7)
   $endDate = (Get-Date)
-  $auditLog = (New-ExoRequest -tenantid $Tenantfilter -cmdlet 'Get-AdminAuditLogConfig').UnifiedAuditLogIngestionEnabled 
+  $auditLog = (New-ExoRequest -tenantid $Tenantfilter -cmdlet 'Get-AdminAuditLogConfig').UnifiedAuditLogIngestionEnabled
   $7dayslog = if ($auditLog -eq $false) {
     $ExtractResult = 'AuditLog is disabled. Cannot perform full analysis'
   } else {
@@ -40,10 +40,10 @@ try {
       Write-Host "Retrieved $($logsTenant.count) logs" -ForegroundColor Yellow
       $logsTenant
     } while ($LogsTenant.count % 5000 -eq 0 -and $LogsTenant.count -ne 0)
-    $ExtractResult = 'Succesfully extracted logs from auditlog'
+    $ExtractResult = 'Successfully extracted logs from auditlog'
   }
   Try {
-    $URI = "https://graph.microsoft.com/beta/auditLogs/signIns?`$filter=(userId eq '$SuspectUser')&`$top=1&`$orderby=createdDateTime desc" 
+    $URI = "https://graph.microsoft.com/beta/auditLogs/signIns?`$filter=(userId eq '$SuspectUser')&`$top=1&`$orderby=createdDateTime desc"
     $LastSignIn = New-GraphGetRequest -uri $URI -tenantid $TenantFilter -noPagination $true -verbose | Select-Object @{ Name = 'CreatedDateTime'; Expression = { $(($_.createdDateTime | Out-String) -replace '\r\n') } },
     id,
     @{ Name = 'AppDisplayName'; Expression = { $_.resourceDisplayName } },

Durable_BECRun/run.ps1

@@ -4,8 +4,7 @@ $Functions = $Public + $Private
 foreach ($import in @($Functions)) {
     try {
         . $import.FullName
-    }
-    catch {
+    } catch {
         Write-Error -Message "Failed to import function $($import.FullName): $_"
     }
 }

Modules/CIPPCore/CIPPCore.psm1

@@ -6,24 +6,25 @@ function Add-CIPPApplicationPermission {
         $Tenantfilter
     )
     if ($ApplicationId -eq $ENV:ApplicationID -and $Tenantfilter -eq $env:TenantID) {
-        return @('Cannot modify application permissions for CIPP-SAM on partner tenant')
+        #return @('Cannot modify application permissions for CIPP-SAM on partner tenant')
+        $RequiredResourceAccess = 'CIPPDefaults'
     }
     Set-Location (Get-Item $PSScriptRoot).FullName
     if ($RequiredResourceAccess -eq 'CIPPDefaults') {
         $RequiredResourceAccess = (Get-Content '.\SAMManifest.json' | ConvertFrom-Json).requiredResourceAccess
     }
-    $ServicePrincipalList = New-GraphGETRequest -uri "https://graph.microsoft.com/beta/servicePrincipals?`$select=AppId,id,displayName&`$top=999" -skipTokenCache $true -tenantid $Tenantfilter
+    $ServicePrincipalList = New-GraphGETRequest -uri "https://graph.microsoft.com/beta/servicePrincipals?`$select=AppId,id,displayName&`$top=999" -skipTokenCache $true -tenantid $Tenantfilter -NoAuthCheck $true
     $ourSVCPrincipal = $ServicePrincipalList | Where-Object -Property AppId -EQ $ApplicationId
     if (!$ourSVCPrincipal) {
         #Our Service Principal isn't available yet. We do a sleep and reexecute after 3 seconds.
         Start-Sleep -Seconds 5
-        $ServicePrincipalList = New-GraphGETRequest -uri "https://graph.microsoft.com/beta/servicePrincipals?`$select=AppId,id,displayName&`$top=999" -skipTokenCache $true -tenantid $Tenantfilter
+        $ServicePrincipalList = New-GraphGETRequest -uri "https://graph.microsoft.com/beta/servicePrincipals?`$select=AppId,id,displayName&`$top=999" -skipTokenCache $true -tenantid $Tenantfilter -NoAuthCheck $true
         $ourSVCPrincipal = $ServicePrincipalList | Where-Object -Property AppId -EQ $ApplicationId
     }
 
     $Results = [System.Collections.ArrayList]@()
 
-    $CurrentRoles = New-GraphGETRequest -uri "https://graph.microsoft.com/beta/servicePrincipals/$($ourSVCPrincipal.id)/appRoleAssignments" -tenantid $Tenantfilter -skipTokenCache $true
+    $CurrentRoles = New-GraphGETRequest -uri "https://graph.microsoft.com/beta/servicePrincipals/$($ourSVCPrincipal.id)/appRoleAssignments" -tenantid $Tenantfilter -skipTokenCache $true -NoAuthCheck $true
 
     $Grants = foreach ($App in $RequiredResourceAccess) {
         $svcPrincipalId = $ServicePrincipalList | Where-Object -Property AppId -EQ $App.resourceAppId
@@ -40,12 +41,13 @@ function Add-CIPPApplicationPermission {
     $counter = 0
     foreach ($Grant in $Grants) {
         try {
-            $SettingsRequest = New-GraphPOSTRequest -body ($Grant | ConvertTo-Json) -uri "https://graph.microsoft.com/beta/servicePrincipals/$($ourSVCPrincipal.id)/appRoleAssignedTo" -tenantid $Tenantfilter -type POST
+            $SettingsRequest = New-GraphPOSTRequest -body (ConvertTo-Json -InputObject $Grant -Depth 5) -uri "https://graph.microsoft.com/beta/servicePrincipals/$($ourSVCPrincipal.id)/appRoleAssignedTo" -tenantid $Tenantfilter -type POST -NoAuthCheck $true
             $counter++
         } catch {
-            $Results.add("Failed to grant $($Grant.appRoleId) to $($Grant.resourceId): $($_.Exception.Message)") | Out-Null
+            $ErrorMessage = Get-NormalizedError -Message $_.Exception.Message
+            $Results.add("Failed to grant $($Grant.appRoleId) to $($Grant.resourceId): $ErrorMessage") | Out-Null
         }
     }
     "Added $counter Application permissions to $($ourSVCPrincipal.displayName)"
     return $Results
-}
+}

Modules/CIPPCore/Public/Add-CIPPApplicationPermission.ps1

@@ -6,54 +6,139 @@ function Add-CIPPAzDataTableEntity {
         [switch]$Force,
         [switch]$CreateTableIfNotExists
     )
-    
+
+    $MaxRowSize = 500000 - 100 # Maximum size of an entity
+    $MaxSize = 30kb # Maximum size of a property value
+
     foreach ($SingleEnt in $Entity) {
         try {
-            Add-AzDataTableEntity -context $Context -force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $SingleEnt -ErrorAction Stop
+            Add-AzDataTableEntity -Context $Context -Force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $SingleEnt -ErrorAction Stop
         } catch [System.Exception] {
-            if ($_.Exception.ErrorCode -eq 'PropertyValueTooLarge' -or $_.Exception.ErrorCode -eq 'EntityTooLarge') {
+            if ($_.Exception.ErrorCode -eq 'PropertyValueTooLarge' -or $_.Exception.ErrorCode -eq 'EntityTooLarge' -or $_.Exception.ErrorCode -eq 'RequestBodyTooLarge') {
                 try {
-                    $MaxSize = 30kb
-                    $largePropertyName = $null
+                    $largePropertyNames = [System.Collections.ArrayList]::new()
+                    $entitySize = 0
                     foreach ($key in $SingleEnt.Keys) {
-                        if ($SingleEnt[$key].Length -gt $MaxSize) {
-                            $largePropertyName = $key
-                            break
+                        $propertySize = [System.Text.Encoding]::UTF8.GetByteCount($SingleEnt[$key].ToString())
+                        $entitySize = $entitySize + $propertySize
+                        if ($propertySize -gt $MaxSize) {
+                            $largePropertyNames.Add($key)
                         }
+
                     }
 
-                    if ($largePropertyName) {
-                        $dataString = $SingleEnt[$largePropertyName]
-                        $splitCount = [math]::Ceiling($dataString.Length / $MaxSize)
-                        $splitData = 0..($splitCount - 1) | ForEach-Object {
-                            $start = $_ * $MaxSize
-                            $dataString.Substring($start, [Math]::Min($MaxSize, $dataString.Length - $start))
-                        }
+                    if ($largePropertyNames.Count -gt 0) {
+                        $splitInfoList = [System.Collections.ArrayList]@()
+                        foreach ($largePropertyName in $largePropertyNames) {
+                            $dataString = $SingleEnt[$largePropertyName]
+                            $splitCount = [math]::Ceiling($dataString.Length / $MaxSize)
+                            $splitData = [System.Collections.ArrayList]@()
+                            for ($i = 0; $i -lt $splitCount; $i++) {
+                                $start = $i * $MaxSize
+                                $splitData.Add($dataString.Substring($start, [Math]::Min($MaxSize, $dataString.Length - $start))) > $null
+                            }
+
+                            $splitPropertyNames = [System.Collections.ArrayList]@()
+                            for ($i = 0; $i -lt $splitData.Count; $i++) {
+                                $splitPropertyNames.Add("${largePropertyName}_Part$i") > $null
+                            }
 
-                        $splitPropertyNames = 1..$splitData.Count | ForEach-Object {
-                            "${largePropertyName}_Part$_"
+                            $splitInfo = @{
+                                OriginalHeader = $largePropertyName
+                                SplitHeaders   = $splitPropertyNames
+                            }
+                            $splitInfoList.Add($splitInfo) > $null
+                            $SingleEnt.Remove($largePropertyName)
+
+                            for ($i = 0; $i -lt $splitData.Count; $i++) {
+                                $SingleEnt[$splitPropertyNames[$i]] = $splitData[$i]
+                            }
                         }
 
-                        $splitInfo = @{
-                            OriginalHeader = $largePropertyName
-                            SplitHeaders   = $splitPropertyNames
+                        $SingleEnt['SplitOverProps'] = ($splitInfoList | ConvertTo-Json -Compress).ToString()
+                    }
+
+                    # Check if the entity is still too large
+                    $entitySize = [System.Text.Encoding]::UTF8.GetByteCount($($SingleEnt | ConvertTo-Json))
+                    if ($entitySize -gt $MaxRowSize) {
+                        $rows = [System.Collections.ArrayList]@()
+                        $originalPartitionKey = $SingleEnt.PartitionKey
+                        $originalRowKey = $SingleEnt.RowKey
+                        $entityIndex = 0
+
+                        while ($entitySize -gt $MaxRowSize) {
+                            Write-Information "Entity size is $entitySize. Splitting entity into multiple parts."
+                            $newEntity = @{}
+                            $newEntity['PartitionKey'] = $originalPartitionKey
+                            if ($entityIndex -eq 0) {
+                                $newEntity['RowKey'] = $originalRowKey
+                            } else {
+                                $newEntity['RowKey'] = "$($originalRowKey)-part$entityIndex"
+                            }
+                            $newEntity['OriginalEntityId'] = $originalRowKey
+                            $newEntity['PartIndex'] = $entityIndex
+                            $entityIndex++
+
+                            $propertiesToRemove = [System.Collections.ArrayList]@()
+                            foreach ($key in $SingleEnt.Keys) {
+                                $newEntitySize = [System.Text.Encoding]::UTF8.GetByteCount($($newEntity | ConvertTo-Json))
+                                if ($newEntitySize -lt $MaxRowSize) {
+                                    $propertySize = [System.Text.Encoding]::UTF8.GetByteCount($SingleEnt[$key].ToString())
+                                    if ($propertySize -gt $MaxRowSize) {
+                                        $dataString = $SingleEnt[$key]
+                                        $splitCount = [math]::Ceiling($dataString.Length / $MaxSize)
+                                        $splitData = [System.Collections.ArrayList]@()
+                                        for ($i = 0; $i -lt $splitCount; $i++) {
+                                            $start = $i * $MaxSize
+                                            $splitData.Add($dataString.Substring($start, [Math]::Min($MaxSize, $dataString.Length - $start))) > $null
+                                        }
+
+                                        $splitPropertyNames = [System.Collections.ArrayList]@()
+                                        for ($i = 0; $i -lt $splitData.Count; $i++) {
+                                            $splitPropertyNames.Add("${key}_Part$i") > $null
+                                        }
+
+                                        for ($i = 0; $i -lt $splitData.Count; $i++) {
+                                            $newEntity[$splitPropertyNames[$i]] = $splitData[$i]
+                                        }
+                                    } else {
+                                        $newEntity[$key] = $SingleEnt[$key]
+                                    }
+                                    $propertiesToRemove.Add($key) > $null
+                                }
+                            }
+
+                            foreach ($prop in $propertiesToRemove) {
+                                $SingleEnt.Remove($prop)
+                            }
+
+                            $rows.Add($newEntity) > $null
+                            $entitySize = [System.Text.Encoding]::UTF8.GetByteCount($($SingleEnt | ConvertTo-Json))
                         }
-                        $SingleEnt['SplitOverProps'] = ($splitInfo | ConvertTo-Json).ToString()
-                        $SingleEnt.Remove($largePropertyName)
 
-                        for ($i = 0; $i -lt $splitData.Count; $i++) {
-                            $SingleEnt[$splitPropertyNames[$i]] = $splitData[$i]
+                        if ($SingleEnt.Count -gt 0) {
+                            $SingleEnt['RowKey'] = "$($originalRowKey)-part$entityIndex"
+                            $SingleEnt['OriginalEntityId'] = $originalRowKey
+                            $SingleEnt['PartIndex'] = $entityIndex
+                            $SingleEnt['PartitionKey'] = $originalPartitionKey
+
+                            $rows.Add($SingleEnt) > $null
                         }
 
-                        Add-AzDataTableEntity -context $Context -force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $SingleEnt
+                        foreach ($row in $rows) {
+                            Write-Information "current entity is $($row.RowKey) with $($row.PartitionKey). Our size is $([System.Text.Encoding]::UTF8.GetByteCount($($row | ConvertTo-Json)))"
+                            Add-AzDataTableEntity -Context $Context -Force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $row
+                        }
+                    } else {
+                        Add-AzDataTableEntity -Context $Context -Force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $SingleEnt
                     }
 
                 } catch {
-                    throw "Error processing entity: $($_.Exception.Message)."
+                    $ErrorMessage = Get-NormalizedError -Message $_.Exception.Message
+                    throw "Error processing entity: $ErrorMessage Linenumber: $($_.InvocationInfo.ScriptLineNumber)"
                 }
             } else {
-                Write-Host "THE ERROR IS $($_.Exception.ErrorCode)"
-
+                Write-Information "THE ERROR IS $($_.Exception.ErrorCode). The size of the entity is $entitySize."
                 throw $_
             }
         }

Modules/CIPPCore/Public/Add-CIPPAzDataTableEntity.ps1

@@ -30,12 +30,12 @@ function Add-CIPPBPAField {
             $Result["$fieldName"] = [bool]$FieldValue
         }
         'JSON' {
-            if ($FieldValue -eq $null) { $JsonString = '{}' } else { $JsonString = (ConvertTo-Json -Depth 15 -InputObject $FieldValue -Compress) }
+            if ($null -eq $FieldValue) { $JsonString = '{}' } else { $JsonString = (ConvertTo-Json -Depth 15 -InputObject $FieldValue -Compress) }
             $Result[$fieldName] = [string]$JsonString
         }
         'string' {
             $Result[$fieldName], [string]$FieldValue
         }
     }
     Add-CIPPAzDataTableEntity @Table -Entity $Result -Force
-}
+}