StoreFront Health Checker

Performs analysis including:
Show bad event log entries for StoreFront
Check config sync if in cluster
Show log file contents
Analyse IIS log files
Check STA URLs
Check bindings and certificates
Version 1.2.8
Created on 2024-07-30
Modified on 2025-01-12
Created by Guy Leech
Downloads: 7

The Script Copy Script Copied to clipboard
#requires -version 3

<#
.SYNOPSIS
    Look for bad event log entries for StoreFront, check config sync if in cluster, show log file contents and check bindings and certificates

.DESCRIPTION
    https://support.citrix.com/article/CTX214759/storefront-failed-to-run-discovery

.PARAMETER daysBack
    How many days back to look for issues

.PARAMETER certificateWarningDays
    Certificates expiring within this many days are added to the warnings

.PARAMETER maximumTimeDifferenceSeconds
    Maximum time difference between clock and authoritative time source before awarning is generated

.PARAMETER webrequestTmeoutSeconds
    Timeout in seconds for web reqests

.NOTES
    Modification History:

    2023/05/31  Guy Leech  Script born
    2023/06/01  Guy Leech  First version released into the wild
    2023/06/05  Guy Leech  Added checks of time against domain and internet
    2023/06/16  Guy Leech  Added event log fetching from DDC and VDA
    2023/12/18  Guy Leech  Try catch around failing get-winevent -computer
    2024/03/07  Guy Leech  Handling of bad domain controllers improved to stop error/hanging
    2024/03/13  Guy Leech  Checking of STA URLs
    2024/03/15  Guy Leech  Optimised IIS log processing. Service checking implemented. IIS error code summary
    2024/07/22  Guy Leech  Added reading of SF XML like log files
    2024/07/25  Guy Leech  Checking app pools, certificate expiry warning parameter added, more IIS error detail
    2024/07/26  Guy Leech  Added checking of StoreFront groups and config sync
    2024/07/31  Guy Leech  Added checking of URLs. Removed repetition of bindings checking for same site. Filter out ::1 source IP for IIS logs
#>

[CmdletBinding()]

Param
(
    [double]$daysBack = 1 ,
    [decimal]$certificateWarningDays = 45 ,
    [int]$maximumTimeDifferenceSeconds = 60 ,
    [int]$webrequestTmeoutSeconds = 30
)

#region ControlUp_Script_Standards
$VerbosePreference = $(if( $PSBoundParameters[ 'verbose' ] ) { $VerbosePreference } else { 'SilentlyContinue' })
$DebugPreference = $(if( $PSBoundParameters[ 'debug' ] ) { $DebugPreference } else { 'SilentlyContinue' })
$ErrorActionPreference = $(if( $PSBoundParameters[ 'erroraction' ] ) { $ErrorActionPreference } else { 'Stop' })
$ProgressPreference = 'SilentlyContinue'

[int]$outputWidth = 250
if( ( $PSWindow = (Get-Host).UI.RawUI ) -and ( $WideDimensions = $PSWindow.BufferSize ) )
{
    try
    {
        $WideDimensions.Width = $outputWidth
        $PSWindow.BufferSize = $WideDimensions
    }
    catch
    {
        ## Nothing we can do but shouldn't cause script to end
    }
}
#endregion ControlUp_Script_Standards

[System.Collections.Generic.List[string]]$warnings = @()
[datetime]$startAnalysisDate = [datetime]::Now.AddDays( -$daysBack )
[string]$logfileNamePattern = 'u_ex*.log'
[string]$providerName = 'Citrix Receiver for Web'
[int]$eventId = 17
$storeFrontInstallKey = Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*' -ErrorAction SilentlyContinue | Where-Object { $_.PSObject.Properties[ 'DisplayName' ] -and $_.DisplayName -imatch '\bCitrix\b.*\bStoreFront\b' -and ( -Not $_.PSObject.Properties[ 'SystemComponent' ] -or $_.SystemComponent -ne 1 ) }
[hashtable]$driveDetails = @{}
[hashtable]$logFolderDetails = @{}
[hashtable]$logFoldersForWebSites = @{}
[hashtable]$bindingsForWebSites = @{}
[hashtable]$checkedSTAs = @{}
[hashtable]$checkedURLs = @{}
[string[]]$expectedServices = @(
    'Citrix Peer Resolution Service'
    'Citrix Cluster Join Service'
    'Citrix Configuration Replication'
    'Citrix Credential Wallet'
    'Citrix Default Domain Services'
    'Citrix Service Monitor'
    'Citrix Subscriptions Store'
    'Citrix Telemetry Service'
)
[string[]]$expectedAppPools = @(
    'Citrix Delivery Services Resources' 
    'Citrix Delivery Services Authentication' 
    'Citrix Receiver for Web' 
    'Citrix Configuration Api'
)
[string[]]$excludedIISProperties = @( 'cs-uri-query' , 'cs-method' , 'cs(User-Agent)' , 'cs(Referer)' , 'time-taken' )
[string]$logsFolder = "$($env:ProgramFiles)\Citrix\Receiver StoreFront\Admin\trace"

## Read SF log files

Function Process-LogFile
{
    Param( [string]$inputFile , [datetime]$start , [datetime]$end )

    [string]$parent = 'GuyLeech' ## doesn't matter what it is since it only ever lives in memory

    Write-Verbose "Processing $inputFile ..." 
    ## Missing parent context so add
    [xml]$wellformatted = "<$parent>" + ( Get-Content -Path $inputFile ) + "</$parent>"

    <#
    <EventID>0</EventID>
      <Type>3</Type>
      <SubType Name="Information">0</SubType>
      <Level>8</Level>
      <TimeCreated SystemTime="2016-05-25T14:59:12.8077640Z" />
      <Source Name="Citrix.DeliveryServices.WebApplication" />
      <Correlation ActivityID="{00000000-0000-0000-0000-000000000000}" />
      <Execution ProcessName="w3wp" ProcessID="52168" ThreadID="243" />
    #>

    try
    {
        $wellformatted.$parent.ChildNodes.Where( { [datetime]$_.System.TimeCreated.SystemTime -ge $start -and [datetime]$_.System.TimeCreated.SystemTime -le $end } ).ForEach( `
        {
            $result = [pscustomobject]@{
                'Date'=[datetime]$_.System.TimeCreated.SystemTime
                'File' = (Split-Path $inputFile -Leaf)
                'Type'=$_.System.Type
                'Subtype'=$_.System.Subtype.Name
                'Level'=$_.System.Level
                'SourceName'=$_.System.Source.Name
                'Computer'=$_.System.Computer
                'ApplicationData'=$_.ApplicationData
            }
            ## Not all objects have all properties
            if( ( Get-Member -InputObject $_.system -Name 'Process' -Membertype Properties -ErrorAction SilentlyContinue ) )
            {
                Add-Member -InputObject $result -NotePropertyMembers @{
                    'Process' = $_.System.Process.ProcessName
                    'PID' =  $_.System.Process.ProcessID
                    'TID' = $_.System.Process.ThreadID
                }
                <#
                $result | Add-Member -MemberType NoteProperty -Name 'Process' -Value $_.System.Process.ProcessName
                $result | Add-Member -MemberType NoteProperty -Name 'PID' -Value $_.System.Process.ProcessID;`
                $result | Add-Member -MemberType NoteProperty -Name 'TID' -Value $_.System.Process.ThreadID;`
                #>
            }
            $result
        } ) | Select-Object -Property Date,File,SourceName,Type,SubType,Level,Computer,Process,PID,TID,ApplicationData ## This is the order they will be displayed in
    }
    catch {}
}

if( -Not $storeFrontInstallKey )
{
    $warnings.Add( "Cannot find installation details for Citrix StoreFront - is it installed or is this the wrong machine?" )
}
else
{
    ## Write-Verbose -Message "StoreFront application id from registry is $($storeFrontInstallKey.PSChildName)"
    Write-Output -InputObject "StoreFront installed version is $($storeFrontInstallKey.DisplayVersion)"
}

[string]$storeFrontConfigurationKeyName = 'HKLM:\SOFTWARE\Citrix\DeliveryServices'

if( $storeFrontConfigurationKey = Get-ItemProperty -Path $storeFrontConfigurationKeyName -ErrorAction SilentlyContinue )
{
    if( $storeFrontConfigurationKey.PSObject.Properties[ 'InstallDir' ] )
    {
        if( Test-Path -Path $storeFrontConfigurationKey.InstallDir )
        {
            [string]$moduleScript = [System.IO.Path]::Combine( $storeFrontConfigurationKey.InstallDir , 'Scripts' , 'ImportModules.ps1' )
            if( Test-Path -Path $moduleScript -PathType Leaf )
            {
                $null = . $moduleScript >$null 4>$null 6>$null ## noisy script so redirect stdout and Write-Host channels to oblivion
                if( -Not $? )
                {
                    $warnings.Add( "Bad status from importing StoreFront PS modules using $moduleScript" )
                }
            }
            else
            {
                $warnings.Add( "Expected StoreFront PowerShell module `"$moduleScript`" does not exist" )
            }
        }
        else
        {
            $warnings.Add( "Installation path `"$($storeFrontConfigurationKey.InstallDir)`" configured in registry does not exist" )
        }
    }
    else
    {
        $warnings.Add( "Unable to find InstallDir value in key $storeFrontConfigurationKeyName" )
    }
}
else
{
    $warnings.Add( "StoreFront configuration key $storeFrontConfigurationKeyName not found" )
}

<#
## Get Citrix Delivery Services event logs - get all because also look for good syncs
[array]$allDeliveryServicesEvents = @( Get-WinEvent -FilterHashtable @{ LogName = 'Citrix Delivery Services' ; StartTime = $startAnalysisDate } -ErrorAction SilentlyContinue | Select-Object -ExcludeProperty Message -Property *,
    ## roll failures for different users and desktop groups into a single error to keep number down without hiding completely
    @{n='Message';e={ $_.Message -replace '(No available resource found for user).*$' , '$1' }} )

[array]$deliveryServiceGroupedEvents = @( $allDeliveryServicesEvents | Group-Object -Property { $_.Level -le 3 } )
$deliveryServiceBadEvents  = @( $deliveryServiceGroupedEvents | Where-Object Name -eq $true  | Select-Object -ExpandProperty Group )
$deliveryServiceGoodEvents = @( $deliveryServiceGroupedEvents | Where-Object Name -eq $false | Select-Object -ExpandProperty Group )
#>

[System.Collections.Generic.List[object]]$deliveryServiceBadEvents  = @()
[System.Collections.Generic.List[object]]$deliveryServiceGoodEvents = @()

[array]$deliveryServiceGoodEvents = @( Get-WinEvent -FilterHashtable @{ LogName = 'Citrix Delivery Services' ; StartTime = $startAnalysisDate } -ErrorAction SilentlyContinue | Select-Object -ExcludeProperty Message -Property *,
    ## roll failures for different users and desktop groups into a single error to keep number down without hiding completely
    @{n='Message';e={ $_.Message -replace '(No available resource found for user).*$' , '$1' }} | ForEach-Object `
    {
        if( $_.Level -le 3 )
        {
            $deliveryServiceBadEvents.Add( $_ )
        }
        else
        {
            $_ ## output to pipeline so goes to deliveryServiceGoodEvents array
        }
    })

if( $null -ne $deliveryServiceBadEvents -and $deliveryServiceBadEvents.Count -gt 0 )
{
    $warnings.Add( "$($deliveryServiceBadEvents.Count) Citrix Event Log Reported Issues:" )

    $warnings += @( $deliveryServiceBadEvents | Group-Object -Property Message | Sort-Object -Property count -Descending | Select-Object -Property count,
        @{name='Last Occurrence';expression={$_.Group.TimeCreated|Sort-Object -Descending|Select-Object -first 1}},
        @{name='Level';expression={$_.Group[0].LevelDisplayName}},
        @{name='Id';expression={$_.Group[0].Id}},
        @{name='Provider';expression={$_.Group[0].ProviderName}},
        @{name='Error' ; expression = { $_.Name -replace "`r?`n" , ' ' -replace '\s{2,}' ,' '}} | Format-Table -AutoSize | Out-String )
}
else
{
    Write-Output -InputObject "No problems found in the Delivery Services Event Log since $($startAnalysisDate.ToString( 'G' ))"
}

##[array]$events = @( Get-WinEvent -FilterHashtable @{ ProviderName = $providerName ; Id = $eventId } -ErrorAction SilentlyContinue )
[array]$events = @( $deliveryServiceBadEvents | Where-Object { $_.ProviderName -ieq $providerName -and $_.Id -eq $eventId })
$OS = $null
$OS = Get-CimInstance -ClassName Win32_OperatingSystem

if( $OS )
{
    Write-Output -InputObject "OS last booted $($OS.LastBootUpTime.ToString('G'))"
}
else
{
    $warnings.Add( "Failed to get OS information" )
}

if( $null -eq $events -or $events.Count -eq 0 )
{
    $providerDetail = Get-WinEvent -ListProvider $providerName -ErrorAction SilentlyContinue
    $oldestEvent = $null
    if( -Not $providerDetail )
    {
        if( -Not $storeFrontInstallKey )
        {
            Throw "Cannot find installation details for Citrix StoreFront - is it installed or is this the wrong machine?"
        }
    }
    else
    {
        $oldestEvent = Get-WinEvent -LogName $providerDetail.LogLinks.LogName -Oldest -MaxEvents 1
    }
    Write-Output -InputObject "No instances of event id $eventId for provider `"$providerName`" found suggesting SSL is ok or server not used"
    if( $oldestEvent )
    {
        Write-Output -InputObject "Oldest event in containing event log is $($oldestEvent.TimeCreated.ToString( 'G' ))"
    }
}
<#
else
{
    Write-Output -InputObject "Most recent of $($events.Count) occurrences of event id $eventId for provider `"$providerName`" was $($events[0].TimeCreated.ToString('G')) - see https://support.citrix.com/article/CTX214759/storefront-failed-to-run-discovery"
}
#>

[array]$citrixServices = @( Get-CimInstance -ClassName win32_service -Filter "Name like 'Citrix%'" )

## have to truncate message because different message depending on whether 1st, 2nd, 3rd, etc failure
[array]$serviceProblems = @( Get-WinEvent -FilterHashtable @{ ProviderName = 'Service Control Manager' ; Level = 1,2,3 ; StartTime = [datetime]::Now.AddDays( -$daysBack ) } -ErrorAction SilentlyContinue |Where-Object Message -match Citrix|Select-object -Property *,@{name='FirstSentence';expression = { $_.Message -replace '\.\s+.*$' }}|group-object firstsentence|select-object count,name,@{name='Most Recent Occurrence';expression={$_.Group.TimeCreated|Sort-Object -Descending|Select-Object -first 1}}|Sort-Object -Property count -Descending )

if( $null -ne $serviceProblems -and $serviceProblems.Count -gt 0 )
{
    $warnings.Add( "Citrix Service Problems" )
    $warnings += @( $serviceProblems | Format-Table -AutoSize | Out-String )
}
else
{
    Write-Output -InputObject "No Citrix service problems found in the event log since $($startAnalysisDate.ToString( 'G' ))"
}

ForEach( $expectedService in $expectedServices )
{
    $thisService = $citrixServices | Where-Object DisplayName -ieq $expectedService
    if( $null -eq $thisService  )
    {
        $warnings.Add( "Service `"$expectedService`" not present" )
    }
    else ## found service so check status
    {
        if( $thisService.StartMode -ieq 'Auto' -and $thisService.State -ine 'Running' )
        {
            $warnings.Add( "Service `"$($thisservice.DisplayName)`" is not running but its start is set to $($thisService.StartMode)" )
        }
        elseif( $thisService.State -ieq 'Running' -and $thisService.Status -ine 'OK' )
        {
            $warnings.Add( "Service `"$($thisservice.DisplayName)`" is running but status is $($thisService.Status)" )
        }
    }
}

$stores = $null
if( Get-Command -Name Get-STFStoreService -ErrorAction SilentlyContinue )
{
    $stores = @( Get-STFStoreService )
    if( $null -eq $stores -or $stores.Count -eq 0 )
    {
         $warnings.Add( "Failed to get store details from StoreFront via PowerShell" )
    }
    else
    {
        Write-Verbose -Message "Got $($stores.Count) stores"
        $stores | Select-Object -Property VirtualPath,ConfigurationFile,@{n='Config Last Written';e={ (Get-ItemProperty -Path $_.ConfigurationFile).LastWriteTime }} | Sort-Object -Property VirtualPath | Format-Table -AutoSize
    }
}
else
{
    $warnings.Add( "Unable to find StoreFront cmdlet Get-STFStoreService" )
}

Import-Module -Name WebAdministration,IISAdministration -Verbose:$false -Debug:$false

$webSites = @( $stores | Select-Object -ExpandProperty WebApplication | Select-Object -ExpandProperty WebSite | Group-Object -Property Name )

Write-Verbose -Message "Stores are using $($webSites.count) web sites"

[string]$matchIISregex = '^(\d|#Fields)'
[string]$replaceIISregex = '^#Fields: '
[array]$includeProperties = @( '*',@{name='duration';expression = { $_.'time-taken' -as [int] }}, @{name='datetime';expression={([datetime]"$($_.date) $($_.time)").ToLocalTime()}} )
[scriptblock]$IISFilterScript = { $_.'s-port' -as [int] -ge 0 -and $_.'c-ip' -ne '127.0.0.1' -and $_.'c-ip' -ne '::1' -and $_.'cs-method' -ieq 'GET' }
[int]$missingIISlogfiles = 0
[int]$IISlogfilesCheckedFor = 0

Write-Verbose -Message "$([datetime]::Now.ToString( 'G')): processing $($websites.count) web sites"

ForEach( $website in $webSites )
{
    $logfilefolder = $null
    $websiteProperties = $null
    [string]$webSiteName = $website.Name
    Write-Verbose -Message "Processing logs from web site $webSiteName"
    $websiteProperties = Get-ItemProperty -Path "IIS:\Sites\$webSiteName" ## .WebApplication.WebSite.Name)"
    $allGetRequests = New-Object -TypeName System.Collections.Generic.List[object]
    [string]$webSiteAndId = "$($webSiteName):$($websiteProperties.Id)"

    if( $websiteProperties.State -ine 'Started' )
    {
        $warnings.Add( "Web site `"$webSiteName`" is not running" )
    }
    if( $websiteProperties.logFile -and $websiteProperties.logFile.enabled )
    {
        $logfilefolder = Join-Path -Path ([environment]::ExpandEnvironmentVariables( $websiteProperties.logFile.directory )) -ChildPath "W3SVC$($websiteProperties.Id)"
        if( -Not ( Test-Path -Path $logfilefolder ) )
        {
            $logfilefolder = $null
        }
        else
        {
            if( -Not $logFoldersForWebSites.ContainsKey( $webSiteAndId ) )
            {
                ## use this when we examine the stores later so we can get the correct log file for that web site & id
                $logFoldersForWebSites.Add( $webSiteAndId , $logfilefolder )
            }
            ## get latest log file and total size
            if( -Not $logFolderDetails.ContainsKey( $logfilefolder ) )
            {
                [string]$logfolderVolumeLetter = $logfilefolder -replace '^(\w).*$' , '$1'
                if( -Not $driveDetails.ContainsKey( $logfolderVolumeLetter ) )
                {
                    $latestLogFile = Get-ChildItem -Path $logfilefolder -File -Force -Recurse -Depth 2 -Filter $logfileNamePattern -ErrorAction SilentlyContinue | Sort-Object -Property LastWriteTime -Descending | Select-Object -First 1
                    [int64]$logfilesTotalSize = (Get-ChildItem -Path $logfilefolder -File -Force -Recurse -Depth 2 -ErrorAction SilentlyContinue | Measure-Object -Sum Length).Sum 
        
                    Write-Output -InputObject "IIS logging is enabled to folder $logfilefolder. Latest log file is $($latestLogFile.Name), last written $($latestLogFile.LastWriteTime.ToString('G')) ($([math]::Round( ([datetime]::Now - $latestLogFile.LastWriteTime).TotalDays , 2)) days ago)"

                    [string]$logfolderVolumeLetter = $logfilefolder -replace '^(\w).*$' , '$1'
                    $logFolderVolume = $null
                    $logFolderVolume = Get-Volume -DriveLetter $logfolderVolumeLetter
                    [string]$message = "IIS log files consuming $([math]::Round( $logfilesTotalSize / 1GB , 2 ))GB on $($logfolderVolumeLetter):"
                    if( $logFolderVolume )
                    {
                        $message += ", free space is $([math]::Round( $logFolderVolume.SizeRemaining / 1GB , 1))GB"
                    }
                    else
                    {
                       $warnings.Add( "Failed to get volume details for volume $logfolderVolumeLetter containing log folder $logfilefolder" )
                    }
                    Write-Output -InputObject $message

                    For( [int]$dayBack = $daysBack ; $daysBack -ge 0 ; $daysBack-- ) 
                    {
                        [datetime]$logFileDate = [datetime]::Today.AddDays( -$daysBack )
                        [string]$logFile = Join-Path -Path $logfilefolder -ChildPath ( $logfileNamePattern -replace '\*' , $logFileDate.ToString('yyMMdd'))
                        Write-Verbose -Message "Processing IIS log file $logfile"
                        $IISlogfilesCheckedFor++
                        if( Test-Path -Path $logFile -PathType Leaf )
                        {
                            $requests = $null
                            try
                            {
                                ## if you change this line, copy to Get-Content one in the catch block - not assigning file contents to variable for speed (could assign to variables & use those)
                                $requests = @( [System.IO.File]::ReadLines( $logfile ) -match $matchIISregex -replace $replaceIISregex  | ConvertFrom-Csv -Delimiter ' ' | Where-Object -FilterScript $IISFilterScript | Select-Object -Property $includeProperties -ExcludeProperty $excludedIISProperties )
                            }
                            catch ## could catch specific "file in use" so we don't retry in other error conditions
                            {
                                try
                                {
                                    Write-Verbose -Message "Error from ReadLines switching to Get-Content on $logfile - $_"
                                    ## copy of ReadLines line from try block
                                    $requests = @( (Get-Content -Path $logfile ) -match $matchIISregex -replace $replaceIISregex  | ConvertFrom-Csv -Delimiter ' ' | Where-Object -FilterScript $IISFilterScript | Select-Object -Property $includeProperties -ExcludeProperty $excludedIISProperties )
                                }
                                catch
                                {
                                    $warnings.Add( "Failed to read IIS log file $logfile - $_" )
                                }
                            }
                            if( $null -ne $requests -and $requests.Count -gt 0 )
                            {
                                $allGetRequests.AddRange( $requests )
                            }
                        }
                        else
                        {
                            $missingIISlogfiles++
                        }
                    }
                }
                ## else already processed this driver / folder
                $logFolderDetails.Add( $logfilefolder , $allGetRequests )
            }
            ## else already processed this folder
            $driveDetails.Add( $logfolderVolumeLetter , $logFolderVolume )
        }
    }
    else
    {
        Write-Output -InputObject "Logging not enabled for web site `"$webSiteName`""
    }
}

if( $missingIISlogfiles -gt 0 )
{
    $warnings.Add( "$missingIISlogfiles / $IISlogfilesCheckedFor IIS log files do not exist" )
}

[array]$appPools = @( Get-IISAppPool | Where-Object -Property Name -match '^Citrix ' )
[string[]]$appPoolNames = @( $appPools | Select-Object -ExpandProperty Name ) ## make it easier to find missing ones

if( $appPools.Count -ne $expectedAppPools.Count )
{
    $missingAppPools = @( ForEach( $appPool in $expectedAppPools ) 
                        {
                            if( $appPoolNames -notcontains $appPool )
                            {
                                $appPool
                            }
                        })
    $warnings.Add( "Expected $($expectedAppPools.Count) Citrix app pools but only found $($appPools.Count). Missing `"$($missingAppPools -join '" , "')`"" )
}

[array]$notStartedAppPools = @( $appPools | Where-Object State -ine 'Started' )
if( $notStartedAppPools.Count -gt 0 )
{
    $warnings.Add( "$($notStartedAppPools.Count) app pools are not started which are `"$(($notStartedAppPools|Select-Object -ExpandProperty Name) -join '" , "')`"" )
}

[array]$IISevents = @( Get-WinEvent -FilterHashtable @{ ProviderName = ( Get-WinEvent -ListProvider *-IIS-* | Select-Object -ExpandProperty Name ) ; Level = 1,2,3 ;StartTime = $startAnalysisDate } -ErrorAction SilentlyContinue | Select-Object -Property *,
    ## group "The Application Host Helper Service encountered an error trying to delete the history directory 'C:\inetpub\history\CFGHISTORY_0000000009'."
   @{ name='Message';expression={ $_.Message -replace '_\d{8,}' , '??' }} -ExcludeProperty Message )

if( $null -ne $IISevents -and $IISevents.Count -gt 0 )
{
    $warnings.Add( "$($IISevents.Count) IIS warning/error events since $($startAnalysisDate.ToString( 'G' ))" )
    $warnings += @( $IISevents | Group-Object -Property Message | Sort-Object -Property count -Descending | Select-Object -Property count,
        @{name='Last Occurrence';expression={$_.Group.TimeCreated | Sort-Object -Descending|Select-Object -first 1}},
        @{name='Level';expression={$_.Group[0].LevelDisplayName}},
        @{name='Id';expression={$_.Group[0].Id}},
        @{name='Provider';expression={$_.Group[0].ProviderName}},
        @{name='Error' ; expression = { $_.Name -replace "`r?`n" , ' ' -replace '\s{2,}' ,' '}}  | Format-Table -AutoSize | Out-String )
}

Write-Verbose -Message "$([datetime]::Now.ToString( 'G')): finished processing web sites, got $($logFolderDetails.count) log folders"

$domain = $null
$domain = [DirectoryServices.ActiveDirectory.Domain]::GetCurrentDomain()
if( $domain )
{
    ## times from domain controllers are UTC
    [int]$counter = 0
    [int]$insync = 0
    [int]$contacted = 0
    [string]$logonDC = $env:LOGONSERVER -replace '^\\\\'
    $timeOnLogonServer = $null
    [double]$worstTimeDifference = 0
    $worstTime = $null
    
    if( $null -eq $domain.DomainControllers-or $domain.DomainControllers.Count -eq 0 )
    {
        $warnings.Add( "No domain controllers returned for domain $($domain.Name) so cannot check for time skew" )
    }
    else
    {
        ForEach( $domainController in $domain.DomainControllers )
        {
            $counter++
            Write-Verbose -Message "Checking DC $($domainController.Name) $counter / $($domain.DomainControllers.Count)"
            $dcTime = $null
            $dcTime = $domainController.CurrentTime
            if( $null -eq $dcTime )
            {
                $warnings.Add( "Failed to get current time from domain controller $($domainController.Name), IP address $($domainController.IPAddress)" )
            }
            else
            {
                $contacted++
                $timeDifferenceSeconds = [math]::Abs( ([datetime]::now.ToUniversalTime() - $domainController.CurrentTime ).TotalSeconds )
                if( $timeDifferenceSeconds -ge $maximumTimeDifferenceSeconds )
                {
                    if( $timeDifferenceSeconds -gt $worstTimeDifference )
                    {
                        $worstTime = $domainController
                        $worstTimeDifference = $timeDifferenceSeconds
                    }
                }
                else
                {
                    $insync++
                }
            }
        }

        if( $contacted -ne $domain.DomainControllers.Count )
        {
            if( $contacted -eq 0 )
            {
                $warnings.Add( "None of the $($domain.DomainControllers.Count) domain controllers could be contacted" )
            }
            else
            {
                $warnings.Add( "Only $contacted of the $($domain.DomainControllers.Count) domain controllers could be contacted" )
            }
        }
        if( $insync -ne $domain.DomainControllers.Count -and $null -ne $worstTime )  ## don't report time if no DCs could be contacted since there are bigger issues
        {
            $warnings.Add( "Time is out with $( ($domain.DomainControllers.Count - $insync) / $domain.DomainControllers.Count * 100)% of $($domain.DomainControllers.Count) domain controllers - worst is $worstTimeDifference seconds on $worstTime" )
        }
    }
    [string]$proxy = $null
    [hashtable]$proxyParameter = @{ }
    [bool]$retry = $true
    $nowUTC = $null
    $utcTime = $null
    [string]$timeURI = 'https://timeapi.io/api/Time/current/zone?timeZone=UTC'

    While( $retry ) 
    {
        try
        {
            $nowUTC = [datetime]::Now.ToUniversalTime()
            $utcTime = Invoke-RestMethod -Uri $timeURI -TimeoutSec $webrequestTmeoutSeconds -UseBasicParsing @proxyParameter
            $retry = $false
        }
        catch
        {
            ## see if there is a proxy at the system level and try that
            if( -Not $proxy -and ( [string[]]$proxies = @( (netsh.exe winhttp show proxy | Select-String -SimpleMatch 'Proxy Server(s)') -split ':' | Select-Object -Skip 1 ) ) -and $proxies.Count )
            {
                $proxy = $proxies[0]
                if( $proxies[0] -notmatch 'https?:' )
                {
                    $proxy = "http://" + $proxies[0].Trim()
                }
                if( $proxies.Count -gt 1 -and ! [string]::IsNullOrEmpty( $proxies[1] ) )
                {
                    $proxy =  "$($proxy):$($proxies[1])"
                }
                Write-Verbose -Message "Trying proxy $proxy"
                $proxyParameter.Add( 'Proxy' , $proxy )
                $proxyParameter.Add( 'ProxyUseDefaultCredentials' , $true ) ## we don't have any others we can pass
            }
        }
    }
    
    if( $utcTime )
    {
        [datetime]$internetTimeUTC = $utcTime.dateTime
        [int]$secondsDifference = [math]::Abs( ($internetTimeUTC - $nowUTC).TotalSeconds )
        Write-Verbose -Message "UTC time locally $($nowUTC.ToString('G')) versus internet UTC $($internetTimeUTC.ToString('G')) - difference $secondsDifference"
        if( $secondsDifference -ge $maximumTimeDifferenceSeconds )
        {
            $warnings.Add( "Time locally is $secondsDifference seconds adrift compared with time from internet" )
        }
    }
    else
    {
        $warnings.Add( "Failed to get time from $timeURI" )
    }
}
else
{
    $warnings.Add( "Failed to get current domain" )
}

[hashtable]$checkedControllers = @{}

$ddcEvents = New-Object -TypeName System.Collections.Generic.List[object]

$configSyncDetails = $null
$storeFrontGroupDetails = $null
$storeFrontGroupDetails = Get-STFServerGroup
if( $null -eq $storeFrontGroupDetails )
{
    $warnings.Add( "Failed to get details for StoreFront server group" )
}
else
{
    Write-Output -InputObject "Got $($storeFrontGroupDetails.ClusterMembers.Count) members in StoreFront server group"
}
if( $storeFrontGroupDetails.ClusterMembers.Count -gt 1 )
{
    ## check connectivity to each server in cluster
     ForEach( $member in $storeFrontGroupDetails.HostBaseUrl )
     {
        Write-Verbose -Message "Testing $($member.DnsSafeHost):$($member.port)"
        $networkConnectionResult = $null
        $networkConnectionResult = Test-NetConnection -ComputerName $member.DnsSafeHost -Port $member.port -InformationLevel Detailed -ErrorAction SilentlyContinue -WarningAction SilentlyContinue
        if( -Not $networkConnectionResult -or -Not $networkConnectionResult.TcpTestSucceeded )
        {
            $warnings.Add( "Failed to contact StoreFront cluster member at $($member.DnsSafeHost):$($member.port)" )
        }
    }
    $configSyncDetails = Get-DSConfigurationReplicationState
    if( $null -ne $configSyncDetails )
    {
        if( $configSyncDetails.LastUpdateStatus -ine 'completed' )
        {
            if( $null -ne $configSyncDetails.ServerStates -and $configSyncDetails.ServerStates.Count -gt 0 )
            {
                ForEach( $state in $configSyncDetails.ServerStates )
                {
                    if( $state.LastUpdateStatus -ine 'completed' )
                    {
                        $warnings.Add( "Last configuration sync with $($state.Servername) was not successful at $($state.LastStartTime.ToString('G')), error was `"$($state.LastError)`"" )
                    }
                }
            }
            else ##no detailed error
            {
                $warnings.Add( "Last configuration sync was not successful at $($configSyncDetails.LastStartTime.ToString('G')), error was `"$($configSyncDetails.LastError)`"" )
            }
        }
        else
        {
            Write-Output -InputObject "Last configuration sync was successful at $($configSyncDetails.LastStartTime.ToString('G')) taking $([math]::round( ($configSyncDetails.LastEndTime - $configSyncDetails.LastStartTime).TotalSeconds,2 )) seconds)"
        }
    }
    else
    {
        $warnings.Add( "Failed to get StoreFront replication state" )
    }
    [array]$installedVersions = @( ForEach( $server in $storeFrontGroupDetails.ClusterMembers )
    {
        $version = $null
        $versionError = $null
        $version = Get-DSConfigurationReplicationVersion -Hostname $server.hostname -ErrorVariable versionError
        [pscustomobject]@{
            Server = $server.hostname
            Version = $version
            Error = $versionError
        }
    })
    [array]$differentVersions = @( $installedVersions | Group-Object -Property Version )
    if( $null -ne $differentVersions -and $differentVersions.Count -gt 1 )
    {
       [int]$badCount = 0
       $failures = $differentVersions | Where-Object { [string]::IsNullOrEmpty( $_.Name ) } ## where version number was null
       if( $null -ne $failures )
       {
            $badCount++
            ForEach( $failure in $failures.Group )
            {
                $warnings.Add( "Failed to get StoreFront version from $($failure.Server), error `"$($failure.Error)`"" )
            }
        }
        if( $differentVersions.Count -ge $badCount )
        {
            $warnings.Add( "Got $($differentVersions.Count - $badCount) different StoreFront versions in cluster across $( ($differentVersions | Measure-Object -Property Count -Sum).Sum - $badCount) servers - $( ($differentVersions | Where-object { -Not [string]::IsNullOrEmpty( $_.Name ) } ).Name -join ' , ')" )
        }
    }
}
else
{
    $warnings.Add( "StoreFront not clustered" )
}
    
ForEach( $store in $stores )
{
    $webSiteAndId = "$($store.WebApplication.WebSite.Name):$($store.WebApplication.WebSite.Id)"
    $logfilefolder = $logFoldersForWebSites[ $webSiteAndId ]
    if( $null -ne $logfilefolder -and ($allGetRequests = $logFolderDetails[ $logfilefolder ] ))
    {
        [array]$requests = ( $allGetRequests | Where-Object 'cs-uri-stem' -match $store.VirtualPath )

        if( $null -eq $requests -or $requests.Count -eq 0 )
        {
            Write-Output -InputObject "No IIS requests matching virtual path $($store.VirtualPath) found in log file $($latestLogFile.Name)"
        }
        else
        {
            [array]$sourceIPs = @( $requests | Group-Object -Property 'c-ip' )
            [array]$errorRequests = @( $requests | Where-Object sc-status -ge 400 )

            Write-Output -InputObject "Got $($requests.Count) IIS requests matching virtual path $($store.VirtualPath) from all log files from $($sourceIPs.Count) different IPs and $($errorRequests.Count) error responses"
            if( $null -ne $errorRequests -and $errorRequests.Count -gt 0 )
            {
                ## errors are ordered so we can get date of first and last from those elements in the group array for each error
                [array]$groupedErrors = @( $errorRequests | Group-Object -Property 'sc-status' | Sort-Object -Property Count -Descending | Select-Object -Property Count,
                    @{name = 'Error' ;expression = {$_.Name}} ,
                    @{name = 'First Occurence' ; expression = { $_.Group[0].datetime.ToString('G') }} ,
                    @{name = 'Last Occurence' ; expression = { if( $_.group.Count -gt 1 ) { $_.Group[-1].datetime.ToString('G') } else { '-' } }} ,
                    @{name = 'Duration (h)' ; expression = { if( $_.group.Count -gt 1 ) { [math]::round( ( $_.Group[-1].datetime - $_.Group[0].datetime ).TotalHours , 1 ) } else { '-' } }} )
                $warnings.Add( "Error summary for $($errorRequests.Count) errors for $($store.VirtualPath):" )
                $warnings += @( $groupedErrors | Format-Table -AutoSize | Out-String  )
            }
        }
    }
    else
    {
        $warnings.Add( "No matching IIS logs for virtual path $($store.VirtualPath)" )
    }

    ## check URLs reachable - not the greatest of tests since we are local but better than nothing
    $store.Routing.ExternalEndpoints | Where-Object Id -ieq 'WebUI' | Select-Object -ExpandProperty Url -Unique -PipelineVariable storeURL | ForEach-Object `
    {
        $response = $null
        $exception = $null
        if( -Not $checkedURLs.ContainsKey( $storeURL ) )
        {
            [string]$warning = $null
            Write-Verbose -Message "Checking URL $storeURL for store $($store.Name)"
            try
            {
                $response = Invoke-WebRequest -UseBasicParsing -Uri $storeURL -TimeoutSec $webrequestTmeoutSeconds
            }
            catch
            {
                $exception = $_.Exception
            }
            if( $null -ne $exception )
            {
                $warning = "Error from URL $storeURL for store $($store.Name) - $($exception|Select-Object -ExpandProperty Message)" 
            }
            elseif( $null -eq $response )
            {
                $warning = "No response or error from URL $storeURL for store $($store.Name)"
            }
            elseif( $response.StatusCode -ge 400 )
            {
                $warning = "Error from URL $storeURL for store $($store.Name) - $($response.StatusCode)"
            }
            ## else good but should we check for some specific content in case redirectedd to an internal page not found page or similar ?
            if( -Not [string]::IsNullOrEmpty( $warning ) )
            {
                $warnings.Add( $warning )
            }
            $checkedURLs.Add( $storeURL , $warning ) ## add regardless of success or fail since we test with ContainsKey so that we only test once
        }
        ## else previously reported so don't repeat
    }

    ## check controllers for stores can be resolved and ports connected to
    [int]$controllersChecked = 0
    ForEach( $farm in $store.FarmsConfiguration.Farms )
    {
        ## check resolution
        ## check port connectivity
        ForEach( $server in $farm.servers )
        {
            [string]$serverAndPort = "$($server):$($farm.Port)"
            if( -Not ( $networkConnectionResult = $checkedControllers[ $serverAndPort ] ) )
            {
                Write-Verbose -Message "Testing $server on port $($farm.port) for store $($store.name)"
                $networkConnectionResult = $null
                $networkConnectionResult = Test-NetConnection -ComputerName $server -Port $farm.port -InformationLevel Detailed -ErrorAction SilentlyContinue -WarningAction SilentlyContinue
                $controllersChecked++
                $checkedControllers.Add( $serverAndPort , $networkConnectionResult )
                if( $daysBack -gt 0 )
                {
                    [datetime]$startEventSearchTime = [datetime]::Now.AddDays( -$daysBack )
                    Write-Verbose -Message "Checking $server for events from $server from $($startEventSearchTime.ToString('G'))"
                    $events = @()
                    try
                    {
                        $events = @( Get-WinEvent -ComputerName $server -FilterHashtable @{ LogName = 'Application' ; StartTime = $startEventSearchTime } -ErrorAction SilentlyContinue )
                        $ddcEvents += $events
                        Write-Verbose -Message "Got $($events.Count) events from $server since $($startEventSearchTime.ToString('G'))"
                    }
                    catch
                    {
                        Write-Verbose -Message "Exception getting events from $server - $_"
                    }
                }
            }
            if( -Not $networkConnectionResult -or -Not $networkConnectionResult.TcpTestSucceeded )
            {
                [string]$pingResult = 'unknown'
                if( $networkConnectionResult )
                {
                    if( $networkConnectionResult.PingSucceeded )
                    {
                        $pingResult = 'worked'
                    }
                    else
                    {
                        $pingResult = 'failed'
                    }
                }
                $warnings.Add( "Unable to contact delivery controller $server (resolved to $($networkConnectionResult.ResolvedAddresses -join ' ')) on port $($farm.port), transport $($farm.TransportType). Ping $pingResult" )
                if( -Not ( $server -as [ipaddress] ))
                {
                    $resolvedTo = $null
                    if( $networkConnectionResult )
                    {
                        $resolvedTo = $networkConnectionResult.AllNameResolutionResults | Select-Object -ExpandProperty IPAddress
                    }
                    else
                    {
                        $resolvedTo = Resolve-DnsName -Name $server -DnsOnly -ErrorAction SilentlyContinue | Select-Object -ExpandProperty IPAddress
                    }
                    if( $null -ne $resolvedTo )
                    {
                        Write-Output -InputObject "$server resolved to $($resolvedTo -join ' ')"
                    }
                    else
                    {
                        $warnings.Add( "Unable to DNS resolve $server" )
                    }
                }
            }
        }

        ## check STA if present
        if( $store.Gateways -and $store.Gateways.Count -gt 0 )
        {
            [string[]]$staURLs = @( $store.Gateways.SecureTicketAuthorityUrls | Select-Object -ExpandProperty StaURL -Unique )
            Write-Verbose -Message "Got $($staURLs.Count) STA URLS for store `"$($store.Name)`""
            ForEach( $staURl in $staURLs )
            {
                $response = $null
                $exception = $null
                $warning = $checkedSTAs[ $staURL ] ## don''t make [string] as $null becomes empty string which is special to us (tested previously and was ok)
                if( $null -eq $warning )
                {
                    Write-Verbose -Message "Checking STA URL $staurl for store $($store.Name)"
                    try
                    {
                        $response = Invoke-WebRequest -UseBasicParsing -Uri $staURl -TimeoutSec $webrequestTmeoutSeconds
                    }
                    catch
                    {
                        $exception = $_.Exception
                    }
                    if( $null -eq $response )
                    {
                        ## we are expecting a 406 error
                        if( -Not $exception )
                        {
                            $warning = "No response from STA URL $staURl for store $($store.Name) and no error" 
                        }
                        elseif( -Not $exception.PSobject.Properties[ 'Response' ] -or -Not $exception.Response -or -Not $exception.Response.PSobject.Properties[ 'StatusCode' ] -or $exception.Response.StatusCode -ine 'NotAcceptable' )
                        {
                            $warning = "Error from STA URL $staURl for store $($store.Name) - $($exception.Message)" 
                        }
                        else ## good response so store empty string so doesn't check again but no message output
                        {
                            $warning = ''
                        }
                        $checkedSTAs.Add( $staURl , $warning )
                    }
                    else ## good response but probably shouldn't have been
                    {
                        $warning = "No error from STA URL $staURl for store $($store.Name) but expected one" 
                    }
                    if( -Not [string]::IsNullOrEmpty( $warning ) )
                    {
                        $warnings.Add( $warning )
                    }
                }
                ## else already warned so don't repeat
            }
        }
    }
    
    if( $controllersChecked -eq 0 )
    {
        $warnings.Add( "No delivery controllers checked for store $($store.Name), check configured correctly!" )
    }

    if( $null -eq $store.WebApplication.WebSite.Bindings -or $store.WebApplication.WebSite.Bindings.Count -eq 0 )
    {
        $warnings.Add( "No bindings at all found for store $($store.Name)" )
        continue
    }

    ## TODO check dates versus when cert last changed if possible - check dates of last write to web.config for SF

    [int]$counter = 0

    Write-Output -InputObject "$($store.WebApplication.WebSite.Bindings.Count) bindings found for StoreFront store $($store.name)"

    ## only check if not already checked since at the web site level so if only 1 web site, only 1 set of bindings
    if( -Not ( $bindingsForWebSites.ContainsKey( $webSiteAndId )))
    {
        ForEach( $binding in $store.WebApplication.WebSite.Bindings )
        {
            if( $binding.HostSource -ieq 'Certificate' -and $null -eq $binding.Certificate )
            {
                $warnings.Add( "Certificate for store $($store.name) missing for binding on port $($binding.port)" )
                continue
            }
            if( $binding.port -eq 443 -and $null -eq $binding.Certificate )
            {
                $warnings.Add( "Binding is on port $($binding.port) but there is no certificate configured for store $($store.name)" )
                continue
            }
        
            if( $storeFrontCertificate = $binding.Certificate )
            {     
                $counter++

                [string]$subject = ( $storeFrontCertificate | Select-Object -ExpandProperty Subject ) -replace '^CN='

                if( [string]::IsNullOrEmpty( $subject ) )
                {
                    $warnings.Add(  "Certificate with thumbprint $($storeFrontCertificate.Thumbprint) for store $($store.name) has no subject" )
                }
                if( $storeFrontCertificate.NotBefore -gt [datetime]::Now )
                {
                    $warnings.Add(  "Certificate with thumbprint $($storeFrontCertificate.Thumbprint) not valid until $($storeFrontCertificate.NotBefore.ToString('G')) " )
                }
                if( $storeFrontCertificate.NotAfter -le [datetime]::Now )
                {
                    $warnings.Add(  "Certificate with thumbprint $($storeFrontCertificate.Thumbprint) expired $($storeFrontCertificate.NotAfter.ToString('G'))" )
                }
                else
                {
                    [int]$daysToExpiry = ($storeFrontCertificate.NotAfter - [datetime]::Now).TotalDays
                    [string]$message = "Certificate with thumbprint $($storeFrontCertificate.Thumbprint) expires on $($storeFrontCertificate.NotAfter.ToString('G')) which is $daysToExpiry days from now"
                    if( $daysToExpiry -le $certificateWarningDays )
                    {
                        $warnings.Add( $message )
                    }
                    else
                    {
                        Write-Output -InputObject $message
                    }
                }
                $matchingSubjectAlternativeName = $storeFrontCertificate.DnsNameList | Where-Object Unicode -ieq $subject
                if( $null -eq $matchingSubjectAlternativeName )
                {
                    $warnings.Add(  "None of the $($storeFrontCertificate.DnsNameList.Count) Subject Alternative Name DNS names match the subject `"$subject`" (thumbprint $($storeFrontCertificate.Thumbprint))" )
                }
                else
                {
                    Write-Output -InputObject "Certificate Subject Alternative Name DNS names are ok ($($matchingSubjectAlternativeName.Unicode))"
                }
            }
            elseif( $binding.BaseUrl -match '^https:' )
            {
                $warnings.Add(  "Store $($store.Name) has a binding on port $($binding.port) for $($binding.BaseURL) with no certificate" )
            }
        }
        $bindingsForWebSites.Add( $webSiteAndId , $true ) ## mark as already done for this website
    }
}
## else already reported on these bindings

[string]$logsFolder = "$($env:ProgramFiles)\Citrix\Receiver StoreFront\Admin\trace" ## default but will set later if we have it in the SF config reg key
if( $storeFrontConfigurationKey -and $storeFrontConfigurationKey.PSObject.Properties[ 'InstallDir' ] )
{
    $logsFolder = Join-Path -Path $storeFrontConfigurationKey.InstallDir -ChildPath '\Admin\trace'
}

if( -Not ( Test-Path -Path $logsFolder -PathType Container ) )
{
    $warnings.Add( "StoreFront logs folder `"$logsFolder`" does not exist" )
}

[int]$filesCount = 0
[array]$relevantLogFiles =  @( Get-ChildItem -Path $logsFolder | Where-Object LastWriteTime -ge $startAnalysisDate )

Write-Verbose -Message "Got $($relevantLogFiles.Count) log files modified after $($startAnalysisDate.ToString('G'))"

[array]$logEntries = @( ForEach( $XMLlogFile in $relevantLogFiles )
{
    $filesCount++
    Write-Verbose -Message "$filesCount / $($relevantLogFiles.Count) : $($XMLlogFile.Name)"
    Process-LogFile -inputFile $XMLlogFile.FullName -start $startAnalysisDate -end ([datetime]::Now) | Where-Object Subtype -ine 'Information'
})

if( $null -ne $logEntries -and $logEntries.Count -gt 0 )
{
    ## remove GUIDs and time spans of form 00:00:00.12345. Replace CRLF separator with space
    $groupedLogEntries = @( $logentries|Select-Object -Property Date,SourceName,Subtype,@{n='Message';e={$_.ApplicationData -replace '\n' -replace  '[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}' -replace '[\s'']\d\d:\d\d:\d\d\.\d+' -replace "`r?`n" , ' ' }} | Group-Object -property message | ForEach-Object `
    {
        $group = $_
        $timerange = $group.group | Measure-Object -Minimum -Maximum -Property Date
        Add-Member -InputObject $group -PassThru -NotePropertyMembers @{
            'First Occurrence' = $timerange.Minimum.ToString('G')
            'Last Occurrence'  = $(if( $group.Count -gt 1 ) { $timerange.Maximum.ToString('G') } else { '-' })  ## only 1 occurence so don't clutter output with what will be the same date
            'Span (h)' = $(if( $group.Count -gt 1 ) { [math]::round( ($timerange.Maximum -$timerange.Minimum).TotalHours , 2 ) } else { '-' })
        }
    })
    $warnings.Add( "$($logEntries.Count) non-information level entries found in $($relevantLogFiles.Count) StoreFront log files modified since $($startAnalysisDate.ToString('G'))" )
    ## don't wrap as have multi line messages
    $warnings += @( $groupedLogEntries | Sort-Object -Property Count -Descending | Select-Object -Property Count, 'First Occurrence' , 'Last Occurrence' , 'Span (h)' ,
        @{name='Error' ; expression = { $_.Name -replace "`r?`n" , ' ' -replace '\s{2,}' ,' '}}  | Format-Table -AutoSize | Out-String )
}

## invoke FQDN, check result

## check config replication in HKEY_LOCAL_MACHINE\SOFTWARE\Citrix\DeliveryServices\ConfigurationReplication

## last modified time of C:\inetpub\wwwroot\Citrix\<store>\App_Data\default.ica

if( $warnings.Count -gt 0 )
{
    $warnings | Write-Warning
}
else
{
    Write-Output -InputObject "No warnings to report"
}