107 lines
3.8 KiB
PowerShell
107 lines
3.8 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$RepeatedSampleComparison,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$LongWindowComparison,
|
|
|
|
[string]$OutputReport
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
function Read-ComparisonReport {
|
|
param([string]$Path, [string]$Label)
|
|
|
|
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
throw "$Label comparison does not exist: $Path"
|
|
}
|
|
$report = Get-Content -Raw -Encoding UTF8 -LiteralPath $Path | ConvertFrom-Json
|
|
if ($null -eq $report -or $null -eq $report.comparisons) {
|
|
throw "$Label comparison has no comparisons array: $Path"
|
|
}
|
|
return $report
|
|
}
|
|
|
|
function Build-ComparisonIndex {
|
|
param([object[]]$Comparisons, [string]$Label)
|
|
|
|
$index = @{}
|
|
foreach ($comparison in $Comparisons) {
|
|
$key = "$($comparison.result)|$($comparison.metric)"
|
|
if ($index.ContainsKey($key)) {
|
|
throw "$Label comparison contains duplicate metric: $key"
|
|
}
|
|
$index[$key] = $comparison
|
|
}
|
|
return $index
|
|
}
|
|
|
|
$repeatedReport = Read-ComparisonReport `
|
|
-Path $RepeatedSampleComparison `
|
|
-Label 'Repeated-sample'
|
|
$longWindowReport = Read-ComparisonReport `
|
|
-Path $LongWindowComparison `
|
|
-Label 'Long-window'
|
|
$repeatedIndex = Build-ComparisonIndex `
|
|
-Comparisons @($repeatedReport.comparisons) `
|
|
-Label 'Repeated-sample'
|
|
$longWindowIndex = Build-ComparisonIndex `
|
|
-Comparisons @($longWindowReport.comparisons) `
|
|
-Label 'Long-window'
|
|
$repeatedKeys = @($repeatedIndex.Keys | Sort-Object)
|
|
$longWindowKeys = @($longWindowIndex.Keys | Sort-Object)
|
|
if (($repeatedKeys | ConvertTo-Json -Compress) -cne ($longWindowKeys | ConvertTo-Json -Compress)) {
|
|
throw 'Comparison metric inventories differ between repeated and long-window protocols'
|
|
}
|
|
|
|
$repeatableRegressions = [System.Collections.Generic.List[object]]::new()
|
|
$nonRepeatableRegressions = [System.Collections.Generic.List[object]]::new()
|
|
foreach ($key in $repeatedKeys) {
|
|
$repeatedMetricComparison = $repeatedIndex[$key]
|
|
$longWindowMetricComparison = $longWindowIndex[$key]
|
|
$repeatedFailed = ([bool]$repeatedMetricComparison.passed) -eq $false
|
|
$longWindowFailed = ([bool]$longWindowMetricComparison.passed) -eq $false
|
|
if ($repeatedFailed -and $longWindowFailed) {
|
|
$repeatableRegressions.Add([pscustomobject]@{
|
|
key = $key
|
|
repeated_sample = $repeatedMetricComparison
|
|
long_window = $longWindowMetricComparison
|
|
})
|
|
} elseif ($repeatedFailed -or $longWindowFailed) {
|
|
$nonRepeatableRegressions.Add([pscustomobject]@{
|
|
key = $key
|
|
failed_protocol = if ($repeatedFailed) { 'repeated_sample' } else { 'long_window' }
|
|
})
|
|
}
|
|
}
|
|
|
|
$summary = [pscustomobject]@{
|
|
schema_version = 1
|
|
repeated_sample_comparison = $RepeatedSampleComparison
|
|
long_window_comparison = $LongWindowComparison
|
|
metric_comparisons = $repeatedKeys.Count
|
|
passed = $repeatableRegressions.Count -eq 0
|
|
repeatable_regressions = @($repeatableRegressions)
|
|
non_repeatable_regressions = @($nonRepeatableRegressions)
|
|
}
|
|
if ($OutputReport) {
|
|
$parent = Split-Path -Parent $OutputReport
|
|
if ($parent -and -not (Test-Path -LiteralPath $parent)) {
|
|
New-Item -ItemType Directory -Path $parent | Out-Null
|
|
}
|
|
$summary | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -LiteralPath $OutputReport
|
|
}
|
|
|
|
if ($repeatableRegressions.Count -gt 0) {
|
|
foreach ($regression in $repeatableRegressions) {
|
|
Write-Error "RENDER_PERFORMANCE_STABILITY: repeatable regression $($regression.key)" -ErrorAction Continue
|
|
}
|
|
Write-Host "RENDER_PERFORMANCE_STABILITY FAIL repeated=$($repeatableRegressions.Count) metrics=$($repeatedKeys.Count)"
|
|
exit 1
|
|
}
|
|
|
|
Write-Host "RENDER_PERFORMANCE_STABILITY PASS metrics=$($repeatedKeys.Count) non_repeatable=$($nonRepeatableRegressions.Count)"
|
|
exit 0
|