# Pacing for the Graph endpoints that allow one request per second. # # Microsoft Graph limits the Conditional Access and identity-protection # resources (conditionalAccessPolicy, namedLocation, riskDetection, riskyUser # and the Conditional Access siblings) to ONE request per second per tenant, # counted across every application that talks to the tenant, and it sends no # Retry-After header when it throttles them. Every $batch sub-request counts on # its own. A list of forty Conditional Access policies fetched twenty per # $batch, or in parallel, therefore throttles at once and then backs off ten # seconds at a time - which the user sees as a hang. # # This file keeps a "last sent" clock per (rate class, tenant) and lets a # caller wait out the remainder of the interval before it sends. The single # request path gates in Invoke-MSGraphAPI; the batch layer keeps paced requests # out of parallel dispatch and posts them one per $batch. The # GraphPaceIdentityEndpoints setting (default on) switches the whole thing off. $script:GraphRateClasses = @( [PSCustomObject]@{ Class = 'ConditionalAccess' Label = 'Conditional Access' IntervalSeconds = 1.0 # Resource paths below the API version, compared case-insensitively. Prefixes = @( 'identity/conditionalAccess/policies', 'identity/conditionalAccess/namedLocations', 'identity/conditionalAccess/authenticationStrengths', 'identity/conditionalAccess/authenticationContextClassReferences', 'identityProtection/riskDetections', 'identityProtection/riskyUsers' ) } ) # (class|tenant) -> [DateTime] UTC of the last send $script:GraphPaceLastSend = @{} function Test-GraphPacingEnabled { return -not ((Get-SettingValue "GraphPaceIdentityEndpoints") -eq $false) } # The pure classifier: the rate class for a URL, or $null when the URL is not # paced. Accepts every shape the engine produces - absolute (nextLinks), # relative with or without a leading slash, with or without the API version, # with query strings and OData key segments. function Resolve-GraphRateClass { param([string]$Url) if([string]::IsNullOrWhiteSpace($Url)) { return $null } $path = $Url.Trim() if($path -match '^https?://[^/]+(/.*)?$') { $path = "$($Matches[1])" } $path = ($path -split '[?#]', 2)[0] $path = $path.TrimStart('/') $path = $path -replace '^(beta|v1\.0)/', '' foreach($rc in $script:GraphRateClasses) { foreach($prefix in $rc.Prefixes) { if(-not $path.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) { continue } # Segment boundary: 'policies' must not match 'policiesTemplates'. if($path.Length -eq $prefix.Length -or $path[$prefix.Length] -eq '/' -or $path[$prefix.Length] -eq '(') { return $rc.Class } } } return $null } # The classifier callers use: honours the setting. function Get-GraphRateClass { param([string]$Url) if(-not (Test-GraphPacingEnabled)) { return $null } return (Resolve-GraphRateClass -Url $Url) } function Get-GraphRateClassInfo { param([string]$Class) foreach($rc in $script:GraphRateClasses) { if($rc.Class -eq $Class) { return $rc } } return $null } # Mockable clock. function Get-GraphPaceNow { return [DateTime]::UtcNow } # The limit is per tenant across all applications, so two tokens for the same # tenant must share one clock. Resolution mirrors the %OrganizationId% # substitution in Invoke-MSGraphAPI; the token id is the last resort. function Get-GraphPaceTenantKey { param($TokenId = 0) try { $provider = Get-AuthProvider if($provider) { $userInfo = $provider.GetUserInfo($TokenId) if($userInfo -and $userInfo.TenantId) { return "$($userInfo.TenantId)" } } } catch { } try { $tenantId = Get-CurrentTenantId if($tenantId) { return "$tenantId" } } catch { } return "token:$TokenId" } # Pull the paced requests out of a batch queue (in place) and return them as a # queue of their own, in their original order. Graph counts every $batch # sub-request on its own and sends no Retry-After for these, so batching them # twenty at a time only throttles at once. Returns an empty queue when the # setting is off. function Split-GraphPacedBatchRequests { param( [System.Collections.Generic.List[PSCustomObject]]$RequestObjects, [string]$BatchType = 'Graph' ) $paced = [System.Collections.Generic.List[PSCustomObject]]::new() if(-not (Test-GraphPacingEnabled)) { return ,$paced } $total = $RequestObjects.Count for($i = $RequestObjects.Count - 1; $i -ge 0; $i--) { if(Resolve-GraphRateClass -Url $RequestObjects[$i].url) { $paced.Insert(0, $RequestObjects[$i]) $RequestObjects.RemoveAt($i) } } if($paced.Count -gt 0) { Write-Log "Batch $($BatchType): $($paced.Count) of $total request(s) target one-per-second endpoints - sending them one at a time after the rest" } # The comma keeps an empty or single-item list from unrolling on return. return ,$paced } # Wait until the tenant's interval for the class has passed since the last # send, then record this send. Call it immediately before the request goes out. function Wait-GraphRatePace { param( [Parameter(Mandatory = $true)][string]$Class, $TokenId = 0 ) $rc = Get-GraphRateClassInfo -Class $Class if(-not $rc) { return } $key = "{0}|{1}" -f $Class, (Get-GraphPaceTenantKey -TokenId $TokenId) $now = Get-GraphPaceNow if($script:GraphPaceLastSend.ContainsKey($key)) { $last = $script:GraphPaceLastSend[$key] $remaining = $rc.IntervalSeconds - ($now - $last).TotalSeconds if($remaining -gt 0) { Write-LogDebug ("{0}: pacing - waiting {1:0.00}s before the next request to the tenant" -f $rc.Label, $remaining) # Sliced and pumped so the window stays alive; no status text of its # own - the callers show their progress. Wait-UIAware -Seconds $remaining -SliceMilliseconds 100 # The send happens right after the wait: never record it earlier # than the interval boundary, even if the clock did not move. $after = Get-GraphPaceNow $boundary = $last.AddSeconds($rc.IntervalSeconds) $now = if($after -gt $boundary) { $after } else { $boundary } } } $script:GraphPaceLastSend[$key] = $now } # Record one retryable batch response against its sub-request budget and say # whether that budget is now spent. Throttling and server errors are counted # separately on purpose: a 429 clears when the rate window rolls over, while a # 5xx that repeats is usually a permanent data error and re-sending it ten times # only freezes the app. $RetryObjects is keyed by batch sub-request id. function Register-GraphRetryAttempt { param( [Parameter(Mandatory = $true)]$Id, [int]$Status, [Parameter(Mandatory = $true)][hashtable]$RetryObjects, [int]$MaxThrottleRetry = 10, [int]$MaxServerErrorRetry = 3, # Optional tally so the caller can report the statuses it actually saw. [hashtable]$StatusCounts ) if($null -ne $StatusCounts) { $key = "$Status" if(-not $StatusCounts.ContainsKey($key)) { $StatusCounts[$key] = 0 } $StatusCounts[$key]++ } $isThrottled = ($Status -eq 429) if(-not $RetryObjects.ContainsKey($Id)) { $RetryObjects[$Id] = @{ Throttle = 0; ServerError = 0 } } $counter = $RetryObjects[$Id] if($isThrottled) { $counter.Throttle++ } else { $counter.ServerError++ } $used = if($isThrottled) { $counter.Throttle } else { $counter.ServerError } $limit = if($isThrottled) { $MaxThrottleRetry } else { $MaxServerErrorRetry } return [PSCustomObject]@{ Attempts = $used; Exhausted = ($used -ge $limit) } } # Render a tally of retryable statuses ({"429" = 2; "500" = 1}) as "429 x2, 500". # Used by the batch dispatcher so its back-off message names the status Graph # actually returned. function Format-GraphRetryStatus { param([hashtable]$StatusCounts) if(-not $StatusCounts -or $StatusCounts.Count -eq 0) { return "a retryable status" } $parts = $StatusCounts.GetEnumerator() | Sort-Object { [int]$_.Key } | ForEach-Object { if($_.Value -gt 1) { "$($_.Key) x$($_.Value)" } else { "$($_.Key)" } } $text = ($parts -join ', ') if($StatusCounts.ContainsKey("429") -and $StatusCounts.Count -eq 1) { return "$text - 'Too many requests'" } return "status $text" } # ---- Retry-After ---- # Moved here from Internal/MSGraph.ps1 with the pacing work: parsing the back-off # header Graph sends belongs beside the clock for the endpoints that never send one. # Normalize an HTTP Retry-After header value into a wait, in seconds. Retry-After can be # delta-seconds (e.g. "120") or an HTTP-date (e.g. "Fri, 31 Jan 2026 23:59:59 GMT"); both # forms are honored. When the header is absent or unparseable we return $DefaultSeconds so # the caller always backs off instead of busy-retrying a throttled/erroring backend. Shared # by the single-request retry (Invoke-MSGraphAPI) and the per-batch-item retry in MSGraph.ps1 so both # treat 429/5xx back-off identically. function Get-GraphRetryAfterSeconds { param( $RetryAfterValue, [int]$DefaultSeconds = 10 ) if($null -eq $RetryAfterValue) { return $DefaultSeconds } $raw = ([string]$RetryAfterValue).Trim() if(-not $raw) { return $DefaultSeconds } # delta-seconds (the form Graph almost always uses) $seconds = 0 if([int]::TryParse($raw, [ref]$seconds)) { if($seconds -gt 0) { return $seconds } return $DefaultSeconds } # HTTP-date form - wait until that instant (invariant culture; header is always GMT) $when = [DateTimeOffset]::MinValue if([DateTimeOffset]::TryParse($raw, [System.Globalization.CultureInfo]::InvariantCulture, [System.Globalization.DateTimeStyles]::AssumeUniversal, [ref]$when)) { $delta = [int][Math]::Ceiling(($when - [DateTimeOffset]::UtcNow).TotalSeconds) if($delta -gt 0) { return $delta } return $DefaultSeconds } return $DefaultSeconds }