IntuneManagement/Internal/BulkExport.ps1

# Internal helpers for Start-GraphBulkExport (Public/Start-GraphBulkExport.ps1).
# Extracted here per architecture rule R11 (keep public driver files thin); these
# are module-internal and never exported. All dot-source into the same module
# scope, so the public driver calls them by name.

# NOTE: the five legacy Sync-BulkExport* policy-hydration helpers (reusable
# settings, branding images, app-config target apps, role-assignment details,
# Terms-of-Use files) were retired. Each now lives on its owning policy class via
# the GetSubResourceBatchRequests / ApplySubResourceBatchResult / FinalizeSubResources
# contract, driven by Internal/PolicySubresourceFetch.ps1 (single + batch + parallel
# in one place). Internal/PolicyHydrateExtras.ps1 was deleted.

# Flag every policy whose export file would land on the same path as another so the
# per-policy GetFileName appends `_<id>` and the duplicate doesn't silently
# overwrite its twin on disk.
#
# Filename uniqueness was previously left to the user-facing AddIDToExportFile
# setting (off by default). Real tenants — one lab tenant has 8 CA policies all named
# "Temp Policy 1" and ~30 SettingsCatalog clones — relied on that setting being on
# to avoid data loss on bulk export. The collision detector kicks in automatically
# so users don't need to discover the setting after losing files.
#
# Two things decide whether two policies collide, and BOTH have to be part of the
# bucket key:
#
#   * The FILE NAME, not the displayName. The two differ whenever sanitization
#     strips a character, and it is the file name that collides on disk. "Wi-Fi:
#     Corp" and "Wi-Fi Corp" are distinct displayNames that both sanitize to
#     "Wi-Fi Corp.json". This always mattered on Windows (':' has always been an
#     invalid file name char) and PortableFileNames extends the same class of
#     collision to Linux and macOS.
#
#   * The FOLDER, which is why this runs once across every type rather than once
#     per type. With AddObjectType off, every type writes straight into the export
#     root, so a per-type pass compared each type only against itself and never
#     noticed a Compliance policy and a Configuration policy of the same name
#     landing on one path. Any two types sharing a .Folder collide the same way
#     even with AddObjectType on.
#
# The name comes from the policy's OWN GetFileName rather than a re-derivation of
# the base transform: subclasses override it (DeviceEnrollmentObject names default
# policies from their id suffix), and a detector that re-implements the base rule
# buckets those by a name they never write under.
function Set-BulkExportFilenameCollisionFlag {
    param(
        [Parameter(Mandatory)]$Policies,
        # Mirrors ExportSettings.AddObjectType: on = a per-type subfolder, off =
        # everything in one directory. Decides whether the type's folder is part of
        # the destination path at all.
        [switch]$AddObjectType
    )

    if (-not $Policies -or @($Policies).Count -lt 2) { return }

    # Every name is read before any flag is set. GetFileName's output depends on
    # _NeedsIdInFilename, so mutating as we go would make later policies in the same
    # bucket hash to their already-disambiguated name and escape detection.
    $byPath = @{}
    foreach ($p in $Policies) {
        if (-not $p) { continue }

        $name = $null
        try { $name = $p.GetFileName($null) }
        catch { Write-LogDebug "Bulk export: could not resolve a file name for '$($p.Name)' - excluded from collision detection: $($_.Exception.Message)" }
        if (-not $name) { continue }

        $folder = if ($AddObjectType) { [string]$p.PolicyType.Folder } else { '' }

        # ToLowerInvariant because the collision is on the file system, and Windows
        # and macOS are case-insensitive: "Baseline.json" and "baseline.json" are one
        # file there. Over-flagging on Linux is harmless - the id suffix is only ever
        # added, never dropped.
        $key = "$folder/$name".ToLowerInvariant()

        if (-not $byPath.ContainsKey($key)) {
            $byPath[$key] = [System.Collections.Generic.List[object]]::new()
        }
        [void]$byPath[$key].Add($p)
    }

    $collided = 0
    foreach ($key in $byPath.Keys) {
        $bucket = $byPath[$key]
        if ($bucket.Count -lt 2) { continue }
        foreach ($p in $bucket) {
            if ($p.PSObject.Properties['_NeedsIdInFilename']) {
                $p._NeedsIdInFilename = $true
                $collided++
            }
        }
    }

    if ($collided -gt 0) {
        Write-Log "Bulk export: $collided policy file(s) resolved to duplicate file names - appending '_<id>' to their filenames to avoid silent overwrite"
    }
}

# ExportFullMembershipPrefixes is gone: all group paths are batched now, so its
# performance rationale went with them, and its only observable output was the
# #DirectMembers / #DirectMemberCount stamps on group sidecars, which nothing in
# the module ever read. An automation run downstream could have read them though,
# so a leftover value must not be dropped in silence - the export would "succeed"
# with the member lists simply absent from the sidecars.
#
# Two places can still hold one, and they need different treatment:
#
#   * A settings file. Read-only from here - rewriting the caller's automation
#     input is not ours to do, so this warns on every run until they edit it.
#
#   * The settings store, where every export used to persist the key alongside
#     AddCompanyName and ExportAssignments. Nothing reads it now, so it is
#     cleared after being reported: otherwise the warning would be permanent for
#     a value the user has no UI left to clear.
#
# The store holds more than one candidate. The old read went through
# Get-SettingValue, which resolves "<tenantId>\IntuneManager" BEFORE the global
# "IntuneManager" path, so a tenant override was the value that actually applied
# and cannot be skipped here. Every applicable path is walked in that same
# precedence order.
#
# A stored empty string reads back the same as "not stored" (Get-SettingStoreValue
# treats "" as unset), so only a non-empty leftover is reportable - which is also
# the only one that ever changed any output. The removal is unconditional
# regardless, so the dead key does not sit in every user's store for ever.
function Clear-BulkExportLegacyMembershipSetting
{
    param(
        $FileSettings,
        [string] $SettingsFile,
        [int] $TokenId = 0
    )

    $removedKey = 'ExportFullMembershipPrefixes'
    $sidecarNote = 'Group sidecars no longer carry the #DirectMembers / #DirectMemberCount member lists.'

    if ($FileSettings -and $FileSettings.PSObject.Properties[$removedKey]) {
        $fileValue = ([string]$FileSettings.$removedKey).Trim()
        if ($fileValue) {
            Write-Log "Bulk export: settings file '$SettingsFile' sets $removedKey ('$fileValue'), which has been removed - the value is ignored. $sidecarNote Remove the property from the settings file to silence this warning." 2
        }
        else {
            # Every file written by an older build carries the key. One that was
            # never set changes nothing, so it is not worth a warning.
            Write-LogDebug "Bulk export: settings file '$SettingsFile' still carries the removed $removedKey property with an empty value - ignored, no change in output."
        }
    }

    # Two tenant ids can be the applicable one: the organization the session is
    # connected to (which is what Get-SettingValue itself keyed on) and, on a
    # cross-tenant automation run, the tenant behind this export's own token.
    $subPaths = [System.Collections.Generic.List[string]]::new()
    $tenantIds = [System.Collections.Generic.List[string]]::new()
    if ($script:OrganizationId) { [void]$tenantIds.Add([string]$script:OrganizationId) }
    $tokenInfo = Get-OperationTokenInfo $TokenId
    if ($tokenInfo -and $tokenInfo.TenantId) { [void]$tenantIds.Add([string]$tokenInfo.TenantId) }

    foreach ($tenantId in $tenantIds) {
        # A path stored flat by an older build ("<tenantId>\IntuneManager" as one
        # property name) resolves from this same string - Get-SettingsTreeNode
        # checks the flat property before splitting the path.
        $tenantPath = "$tenantId\IntuneManager"
        if (-not $subPaths.Contains($tenantPath)) { [void]$subPaths.Add($tenantPath) }
    }
    [void]$subPaths.Add("IntuneManager")   # global last, as it was read last

    foreach ($subPath in $subPaths) {
        $storedValue = Get-SettingStoreValue $subPath $removedKey
        if ($storedValue) {
            Write-Log "Bulk export: the saved setting $subPath\$removedKey ('$storedValue') has been removed - it is ignored and is being cleared from the settings store. $sidecarNote" 2
        }
        # Not inside the if: an empty leftover is worth no warning but is still a
        # dead key. Removal touches the store only when the key is actually there.
        Remove-SettingStoreValue $subPath $removedKey
    }
}

# Pre-warm the AAD-object cache that Add-GraphMigrationObject reads on every
# group reference. Walks three sources visible in the already-fetched policy
# bodies: assignment targets, CA conditions.users.include/excludeGroups, and
# compliance scheduledActionConfigurations.notificationMessageCCList. Without
# this each unique group is fetched one-by-one (sequential Invoke-MSGraphAPI
# calls per policy) during Phase 3, which dominates the wall time for CA-heavy
# or assignment-heavy tenants. Batched here in parallel chunks; subsequent
# per-policy lookups are all cache hits.
function Sync-BulkExportMigrationGroups
{
    param(
        [Parameter(Mandatory)] $Policies,
        [int] $TokenId = 0
    )

    if (-not $Policies -or $Policies.Count -eq 0) { return }

    # Collect unique group ids referenced by any group-style assignment target.
    $groupIds = [System.Collections.Generic.HashSet[string]]::new()
    $groupTargetTypes = @(
        '#microsoft.graph.groupAssignmentTarget'
        '#microsoft.graph.exclusionGroupAssignmentTarget'
        '#microsoft.graph.cloudPcManagementGroupAssignmentTarget'
    )

    # CA condition group ids are special-cased strings, not real group ids.
    $caSkipIds = @('All','None','GuestsOrExternalUsers')

    foreach ($p in $Policies) {
        $body = $p.JsonObject
        if (-not $body) { continue }

        # Assignments live on JsonObject.Assignments after Get-GraphPolicies populates them.
        foreach ($a in @($body.Assignments)) {
            foreach ($target in @($a.target)) {
                if (-not $target -or -not $target.'@odata.type') { continue }
                if ($groupTargetTypes -contains $target.'@odata.type' -and $target.groupId) {
                    [void]$groupIds.Add($target.groupId)
                }
            }
        }

        # CA conditions reference groups by id directly in the policy body. These
        # are resolved by IntuneConditionalAccessClasses.ps1 PostExportCommand —
        # one Graph GET per id during the write phase if not pre-cached here.
        $caUsers = $body.conditions.users
        if ($caUsers) {
            foreach ($id in @($caUsers.includeGroups) + @($caUsers.excludeGroups)) {
                if (-not $id) { continue }
                if ($caSkipIds -contains $id) { continue }
                [void]$groupIds.Add($id)
            }
        }

        # Compliance notification CC groups — resolved by IntuneComplianceClasses
        # PostExportCommand. The expand on scheduledActionConfigurations is on by
        # default for compliance policies, so the ids are already in the body.
        foreach ($rule in @($body.scheduledActionsForRule)) {
            foreach ($cfg in @($rule.scheduledActionConfigurations)) {
                foreach ($id in @($cfg.notificationMessageCCList)) {
                    if ($id) { [void]$groupIds.Add($id) }
                }
            }
        }
    }
    if ($groupIds.Count -eq 0) { return }

    # One token, never the whole list - see the prefetch above.
    $tokenInfo = Get-OperationTokenInfo $TokenId
    if (-not $tokenInfo -or -not $tokenInfo.TenantId) {
        Write-Log "Bulk export: skipping migration-group prefetch - no token info for TokenId $TokenId" 2
        return
    }
    $cacheKey  = "AADObjectCache_$($tokenInfo.TenantId)"
    $cacheFile = "TenantCache_$($tokenInfo.TenantId)"
    $cache     = Get-CacheObject $cacheKey @{}
    if ($cache -isnot [Hashtable]) { $cache = @{} }

    # Skip ids already cached (positive OR negative — null entries mean "known missing").
    $needFetch = [System.Collections.Generic.List[string]]::new()
    foreach ($gid in $groupIds) {
        if (-not $cache.ContainsKey($gid)) { [void]$needFetch.Add($gid) }
    }

    if ($needFetch.Count -gt 0) {
        Write-Log "Bulk export: pre-fetching $($needFetch.Count) AAD group(s) referenced by policies (assignments, CA conditions, compliance notifications)"
        Write-Status -Detail ("Pre-fetching {0} group(s)" -f $needFetch.Count) -SkipLog -Force

        # Full bodies, a thousand ids per POST, through /directoryObjects/getByIds -
        # the same resolver the documentation preload uses. This used to be one
        # GET groups/<id> sub-request per group inside $batch (fifty POSTs for a
        # thousand groups); now it is one or two POSTs. Verified live 2026-09-07:
        # the body is identical to GET groups/<id> in every property and value
        # except an extra @odata.type, which is stripped below so the Groups/
        # sidecars stay byte-identical to before. A group the directory does not
        # return is simply absent from the response: deleted, so it is
        # negative-cached exactly as a 404 was.
        $resolved = 0
        $missing  = 0
        for ($i = 0; $i -lt $needFetch.Count; $i += 1000) {
            $end   = [Math]::Min($i + 999, $needFetch.Count - 1)
            $chunk = @($needFetch[$i..$end])

            $resp = $null
            try {
                $body = (@{ ids = $chunk; types = @('group') } | ConvertTo-Json -Compress)
                $resp = Invoke-MSGraphAPI -Url 'directoryObjects/getByIds' -Content $body -HttpMethod POST -TokenId $TokenId -ODataMetadata 'none'
            }
            catch {
                Write-LogError "Bulk export: getByIds group prefetch failed for a chunk of $($chunk.Count)" $_.Exception
            }
            if ($null -eq $resp) {
                # Transport or auth failure: leave the chunk UNcached so the per-policy
                # path can still resolve those ids later, rather than negative-caching
                # a thousand live groups on one bad call.
                continue
            }

            $seen = [System.Collections.Generic.HashSet[string]]::new()
            foreach ($g in @($resp.value)) {
                if (-not $g -or -not $g.id) { continue }
                if ($g.PSObject.Properties['@odata.type']) { [void]$g.PSObject.Properties.Remove('@odata.type') }
                $cache[[string]$g.id] = $g
                [void]$seen.Add([string]$g.id)
                $resolved++
            }
            foreach ($gid in $chunk) {
                if (-not $seen.Contains($gid)) {
                    $cache[$gid] = $null
                    $missing++
                }
            }
        }
        Set-CacheObject $cacheKey $cache $cacheFile
        Write-Log "Bulk export: migration-group prefetch complete - $resolved resolved, $missing missing/deleted"
    }
}

# Pre-walk the nested-group hierarchy in batches so the per-policy recursion
# inside Add-GraphMigrationObject doesn't fire one un-batched GET per parent
# group plus one per child body. BFS, level-by-level: at each level we batch
# /members/microsoft.graph.group for every group at that level, attach the
# resulting children list to the parent body as `#NestedChildren`, then batch
# /groups/{id} for any newly-discovered children that aren't yet in the
# AADObjectCache. Stops when there are no more groups to walk or when we hit
# MaxDepth-1 (depth 1 means "just the directly-assigned group", so no walk).
#
# The dedup inside Add-GraphMigrationObject's recursion still handles cycles
# correctly — this function is purely a Graph-call optimizer; it doesn't
# change which groups end up in the export.
function Sync-BulkExportNestedGroupHierarchy
{
    param(
        [Parameter(Mandatory)] [int] $MaxDepth,
        [int] $TokenId = 0
    )

    if ($MaxDepth -le 1) { return }

    # One token, never the whole list - see the prefetch above.
    $tokenInfo = Get-OperationTokenInfo $TokenId
    if (-not $tokenInfo -or -not $tokenInfo.TenantId) {
        Write-Log "Bulk export: skipping nested-group hierarchy prefetch - no token info for TokenId $TokenId" 2
        return
    }

    $cacheKey  = "AADObjectCache_$($tokenInfo.TenantId)"
    $cacheFile = "TenantCache_$($tokenInfo.TenantId)"
    $cache     = Get-CacheObject $cacheKey @{}
    if ($cache -isnot [Hashtable]) { $cache = @{} }

    # Seed the walk with every group already in cache (i.e. every group the
    # earlier Sync-BulkExportMigrationGroups resolved). These are the level-1
    # starting points for the hierarchy.
    $currentLevelIds = [System.Collections.Generic.List[string]]::new()
    foreach ($entry in $cache.GetEnumerator()) {
        if ($null -eq $entry.Value) { continue }                                          # negative-cached
        if ($entry.Value.PSObject.Properties.Name -contains '#NestedChildren') { continue } # already walked this session
        [void]$currentLevelIds.Add([string]$entry.Key)
    }

    if ($currentLevelIds.Count -eq 0) { return }

    $totalParents  = 0
    $totalChildren = 0

    # MaxDepth=2 means "walk one level": stamp #NestedChildren on every level-1
    # group. MaxDepth=3 → walk two levels. Loop bound is MaxDepth-1.
    for ($depth = 1; $depth -lt $MaxDepth -and $currentLevelIds.Count -gt 0; $depth++) {

        Write-Log "Bulk export: nested-group hierarchy - fetching children for $($currentLevelIds.Count) group(s) at depth $depth"
        Write-Status -Detail ("Pre-fetching nested groups (depth {0}, {1} parent(s))" -f $depth, $currentLevelIds.Count) -SkipLog -Force

        # Batch-fetch /members/microsoft.graph.group for every parent at this level.
        $memBatch  = [System.Collections.Generic.List[PSCustomObject]]::new()
        $idToParent = @{}
        $i = 0
        foreach ($gid in $currentLevelIds) {
            $i++
            $reqId = "nest_${depth}_$i"
            [void]$memBatch.Add([PSCustomObject]@{
                id      = $reqId
                method  = 'GET'
                url     = "groups/$gid/members/microsoft.graph.group?`$select=id,displayName"
                headers = @{ Accept = 'application/json;odata.metadata=minimal' }
            })
            $idToParent[$reqId] = $gid
        }

        $memResults = Invoke-GraphBatchRequest -BatchObjects $memBatch -BatchType "Nested group members (depth $depth)" -TokenId $TokenId -AllPages -SkipWarnings -IncludedFailed

        # Walk results: stamp #NestedChildren on parents, collect new child ids.
        $newChildIds = [System.Collections.Generic.HashSet[string]]::new()
        $childListsByParent = @{}
        foreach ($r in $memResults) {
            $parentId = $idToParent["$($r.Id)"]
            if (-not $parentId) { continue }

            $children = [System.Collections.Generic.List[PSCustomObject]]::new()
            if ($r.body -and $r.Status -ge 200 -and $r.Status -lt 300) {
                foreach ($c in @($r.body.value)) {
                    if (-not $c -or -not $c.id) { continue }
                    [void]$children.Add([PSCustomObject]@{ id = $c.id; displayName = $c.displayName })
                    if (-not $cache.ContainsKey($c.id)) { [void]$newChildIds.Add($c.id) }
                }
            }
            else {
                Write-LogDebug "Bulk export: nested-group members fetch for $parentId returned HTTP $($r.Status)"
            }

            $parentBody = $cache[$parentId]
            if ($null -ne $parentBody) {
                Add-Member -InputObject $parentBody -MemberType NoteProperty -Name '#NestedChildren' -Value $children.ToArray() -Force
            }
            $childListsByParent[$parentId] = $children
            $totalParents++
        }

        # Batch-fetch bodies for newly-discovered children so the recursion
        # inside Add-GraphMigrationObject hits cache for them too.
        if ($newChildIds.Count -gt 0) {
            Write-Status -Detail ("Pre-fetching {0} nested-group bod(ies) (depth {1})" -f $newChildIds.Count, $depth) -SkipLog -Force

            $bodyBatch  = [System.Collections.Generic.List[PSCustomObject]]::new()
            $idToChild  = @{}
            $j = 0
            foreach ($gid in $newChildIds) {
                $j++
                $reqId = "nestbody_${depth}_$j"
                [void]$bodyBatch.Add([PSCustomObject]@{
                    id      = $reqId
                    method  = 'GET'
                    url     = "groups/$gid"
                    headers = @{ Accept = 'application/json;odata.metadata=none' }
                })
                $idToChild[$reqId] = $gid
            }

            $bodyResults = Invoke-GraphBatchRequest -BatchObjects $bodyBatch -BatchType "Nested group bodies (depth $depth)" -TokenId $TokenId -SkipWarnings -IncludedFailed

            foreach ($r in $bodyResults) {
                $gid = $idToChild["$($r.Id)"]
                if (-not $gid) { continue }
                if ($r.body -and $r.Status -ge 200 -and $r.Status -lt 300) {
                    $cache[$gid] = $r.body
                }
                else {
                    $cache[$gid] = $null  # negative-cache so we don't re-query during the write phase
                }
            }
            $totalChildren += $newChildIds.Count
        }

        # Next iteration walks the children we just found that weren't already in cache
        # and don't yet have their own #NestedChildren stamp.
        $nextLevel = [System.Collections.Generic.List[string]]::new()
        foreach ($pair in $childListsByParent.GetEnumerator()) {
            foreach ($child in $pair.Value) {
                $body = $cache[$child.id]
                if ($null -eq $body) { continue }
                if ($body.PSObject.Properties.Name -contains '#NestedChildren') { continue }
                [void]$nextLevel.Add($child.id)
            }
        }
        $currentLevelIds = $nextLevel
    }

    Set-CacheObject $cacheKey $cache $cacheFile
    Write-Log "Bulk export: nested-group hierarchy prefetch complete - $totalParents parent(s) walked, $totalChildren new child group bod(ies) cached"
}