336 lines
12 KiB
PowerShell
336 lines
12 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string[]]$BaselineReport,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string[]]$CandidateReport,
|
|
|
|
[string]$OutputReport
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
$failures = [System.Collections.Generic.List[string]]::new()
|
|
$comparisons = [System.Collections.Generic.List[object]]::new()
|
|
|
|
function Read-RenderReport {
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$Path,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$Label
|
|
)
|
|
|
|
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
throw "$Label report does not exist: $Path"
|
|
}
|
|
$report = Get-Content -Raw -Encoding UTF8 -LiteralPath $Path | ConvertFrom-Json
|
|
if ($null -eq $report -or $null -eq $report.results) {
|
|
throw "$Label report has no results array: $Path"
|
|
}
|
|
return $report
|
|
}
|
|
|
|
function Convert-ToStableJson {
|
|
param([object]$Value)
|
|
return $Value | ConvertTo-Json -Depth 20 -Compress
|
|
}
|
|
|
|
function Get-MedianValue {
|
|
param([double[]]$Values)
|
|
|
|
$ordered = @($Values | Sort-Object)
|
|
if ($ordered.Count -eq 0) {
|
|
throw 'Cannot calculate a median from an empty value set'
|
|
}
|
|
$middle = [math]::Floor($ordered.Count / 2)
|
|
if ($ordered.Count % 2 -eq 1) {
|
|
return [double]$ordered[$middle]
|
|
}
|
|
return ([double]$ordered[$middle - 1] + [double]$ordered[$middle]) / 2.0
|
|
}
|
|
|
|
function Assert-EqualValue {
|
|
param(
|
|
[string]$Name,
|
|
[object]$BaselineValue,
|
|
[object]$CandidateValue
|
|
)
|
|
|
|
$baselineJson = Convert-ToStableJson $BaselineValue
|
|
$candidateJson = Convert-ToStableJson $CandidateValue
|
|
if ($baselineJson -cne $candidateJson) {
|
|
$failures.Add("$Name differs: baseline=$baselineJson candidate=$candidateJson")
|
|
}
|
|
}
|
|
|
|
function Get-ResultKey {
|
|
param([object]$Result)
|
|
return "$($Result.name)|$($Result.pass)"
|
|
}
|
|
|
|
function Assert-CompatibleCacheInventory {
|
|
param(
|
|
[object]$BaselineInventory,
|
|
[object]$CandidateInventory
|
|
)
|
|
|
|
$baselineNames = @($BaselineInventory.PSObject.Properties.Name | Sort-Object)
|
|
$candidateNames = @($CandidateInventory.PSObject.Properties.Name | Sort-Object)
|
|
Assert-EqualValue 'cache inventory key set' $baselineNames $candidateNames
|
|
foreach ($cacheName in $baselineNames) {
|
|
$baselineRecord = $BaselineInventory.$cacheName
|
|
$candidateRecord = $CandidateInventory.$cacheName
|
|
if ($null -eq $candidateRecord) {
|
|
continue
|
|
}
|
|
Assert-EqualValue "cache inventory presence for $cacheName" `
|
|
$baselineRecord.present `
|
|
$candidateRecord.present
|
|
Assert-EqualValue "cache inventory file count for $cacheName" `
|
|
$baselineRecord.file_count `
|
|
$candidateRecord.file_count
|
|
if ([bool]$baselineRecord.present -and [int64]$candidateRecord.file_count -le 0) {
|
|
$failures.Add("cache inventory is empty for $cacheName")
|
|
}
|
|
}
|
|
}
|
|
|
|
function Build-ResultIndex {
|
|
param(
|
|
[object[]]$Results,
|
|
[string]$Label
|
|
)
|
|
|
|
$index = @{}
|
|
foreach ($result in $Results) {
|
|
$key = Get-ResultKey $result
|
|
if ($index.ContainsKey($key)) {
|
|
$failures.Add("$Label report contains duplicate result: $key")
|
|
continue
|
|
}
|
|
$index[$key] = $result
|
|
}
|
|
return $index
|
|
}
|
|
|
|
function Merge-RenderReports {
|
|
param(
|
|
[object[]]$Reports,
|
|
[string]$Label
|
|
)
|
|
|
|
$merged = Convert-ToStableJson $Reports[0] | ConvertFrom-Json
|
|
if ($Reports.Count -eq 1) {
|
|
return $merged
|
|
}
|
|
$resultIndexes = @(
|
|
foreach ($sample in $Reports) {
|
|
Build-ResultIndex -Results @($sample.results) -Label "$Label sample"
|
|
}
|
|
)
|
|
$referenceResultKeys = @($resultIndexes[0].Keys | Sort-Object)
|
|
for ($sampleNumber = 1; $sampleNumber -lt $Reports.Count; $sampleNumber++) {
|
|
$sample = $Reports[$sampleNumber]
|
|
Assert-EqualValue "$Label[$sampleNumber] schema_version" $merged.schema_version $sample.schema_version
|
|
Assert-EqualValue "$Label[$sampleNumber] profile" $merged.profile $sample.profile
|
|
Assert-EqualValue "$Label[$sampleNumber] cache_state" $merged.cache_state $sample.cache_state
|
|
Assert-EqualValue "$Label[$sampleNumber] viewport" $merged.environment.viewport $sample.environment.viewport
|
|
Assert-EqualValue "$Label[$sampleNumber] Godot version" $merged.environment.godot_version.string $sample.environment.godot_version.string
|
|
Assert-EqualValue "$Label[$sampleNumber] rendering driver" $merged.environment.rendering_driver $sample.environment.rendering_driver
|
|
Assert-EqualValue "$Label[$sampleNumber] rendering method" $merged.environment.rendering_method $sample.environment.rendering_method
|
|
Assert-EqualValue "$Label[$sampleNumber] video adapter" $merged.environment.video_adapter $sample.environment.video_adapter
|
|
Assert-EqualValue "$Label[$sampleNumber] CPU" $merged.environment.cpu $sample.environment.cpu
|
|
Assert-EqualValue "$Label[$sampleNumber] cache contract" $merged.cache_contract $sample.cache_contract
|
|
Assert-EqualValue "$Label[$sampleNumber] cache inventory" $merged.cache_inventory $sample.cache_inventory
|
|
Assert-EqualValue `
|
|
"$Label[$sampleNumber] result key set" `
|
|
$referenceResultKeys `
|
|
@($resultIndexes[$sampleNumber].Keys | Sort-Object)
|
|
}
|
|
|
|
$metricDefinitions = @(
|
|
@('load_time_ms', 'root'),
|
|
@('frame_ms_p95', 'metrics'),
|
|
@('frame_ms_p99', 'metrics'),
|
|
@('max_hitch_ms', 'metrics'),
|
|
@('memory_static_bytes', 'metrics'),
|
|
@('video_memory_bytes', 'metrics')
|
|
)
|
|
$mergedResultIndex = Build-ResultIndex -Results @($merged.results) -Label "Median $Label"
|
|
foreach ($resultKey in $referenceResultKeys) {
|
|
$mergedResult = $mergedResultIndex[$resultKey]
|
|
foreach ($definition in $metricDefinitions) {
|
|
$metricName = $definition[0]
|
|
$location = $definition[1]
|
|
$values = @(
|
|
foreach ($sampleIndex in $resultIndexes) {
|
|
if ($location -eq 'root') {
|
|
[double]$sampleIndex[$resultKey].$metricName
|
|
} else {
|
|
[double]$sampleIndex[$resultKey].metrics.$metricName
|
|
}
|
|
}
|
|
)
|
|
$medianValue = Get-MedianValue -Values $values
|
|
if ($location -eq 'root') {
|
|
$mergedResult.$metricName = $medianValue
|
|
} else {
|
|
$mergedResult.metrics.$metricName = $medianValue
|
|
}
|
|
}
|
|
}
|
|
return $merged
|
|
}
|
|
|
|
function Compare-Metric {
|
|
param(
|
|
[string]$ResultKey,
|
|
[string]$MetricName,
|
|
[double]$BaselineValue,
|
|
[double]$CandidateValue,
|
|
[double]$AllowedRegressionPercent
|
|
)
|
|
|
|
$limit = if ($BaselineValue -eq 0.0) {
|
|
0.0
|
|
} else {
|
|
$BaselineValue * (1.0 + $AllowedRegressionPercent / 100.0)
|
|
}
|
|
$regressionPercent = if ($BaselineValue -eq 0.0) {
|
|
if ($CandidateValue -eq 0.0) { 0.0 } else { [double]::PositiveInfinity }
|
|
} else {
|
|
(($CandidateValue - $BaselineValue) / $BaselineValue) * 100.0
|
|
}
|
|
$passed = $CandidateValue -le $limit
|
|
$comparisons.Add([pscustomobject]@{
|
|
result = $ResultKey
|
|
metric = $MetricName
|
|
baseline = $BaselineValue
|
|
candidate = $CandidateValue
|
|
limit = $limit
|
|
regression_percent = $regressionPercent
|
|
passed = $passed
|
|
})
|
|
if (-not $passed) {
|
|
$failures.Add(
|
|
("{0} {1} regressed by {2:N3}%: baseline={3:N3} candidate={4:N3} limit={5:N3}" -f
|
|
$ResultKey,
|
|
$MetricName,
|
|
$regressionPercent,
|
|
$BaselineValue,
|
|
$CandidateValue,
|
|
$limit)
|
|
)
|
|
}
|
|
}
|
|
|
|
$baselineReports = @(
|
|
for ($baselineIndex = 0; $baselineIndex -lt $BaselineReport.Count; $baselineIndex++) {
|
|
Read-RenderReport `
|
|
-Path $BaselineReport[$baselineIndex] `
|
|
-Label "Baseline[$baselineIndex]"
|
|
}
|
|
)
|
|
$candidateReports = @(
|
|
for ($candidateIndex = 0; $candidateIndex -lt $CandidateReport.Count; $candidateIndex++) {
|
|
Read-RenderReport `
|
|
-Path $CandidateReport[$candidateIndex] `
|
|
-Label "Candidate[$candidateIndex]"
|
|
}
|
|
)
|
|
$baseline = Merge-RenderReports -Reports $baselineReports -Label 'baseline'
|
|
$candidate = Merge-RenderReports -Reports $candidateReports -Label 'candidate'
|
|
|
|
Assert-EqualValue 'schema_version' $baseline.schema_version $candidate.schema_version
|
|
Assert-EqualValue 'profile' $baseline.profile $candidate.profile
|
|
Assert-EqualValue 'cache_state' $baseline.cache_state $candidate.cache_state
|
|
Assert-EqualValue 'viewport' $baseline.environment.viewport $candidate.environment.viewport
|
|
Assert-EqualValue 'Godot version' $baseline.environment.godot_version.string $candidate.environment.godot_version.string
|
|
Assert-EqualValue 'rendering driver' $baseline.environment.rendering_driver $candidate.environment.rendering_driver
|
|
Assert-EqualValue 'rendering method' $baseline.environment.rendering_method $candidate.environment.rendering_method
|
|
Assert-EqualValue 'video adapter' $baseline.environment.video_adapter $candidate.environment.video_adapter
|
|
Assert-EqualValue 'CPU' $baseline.environment.cpu $candidate.environment.cpu
|
|
Assert-EqualValue 'cache contract' $baseline.cache_contract $candidate.cache_contract
|
|
Assert-CompatibleCacheInventory $baseline.cache_inventory $candidate.cache_inventory
|
|
|
|
$baselineIndex = Build-ResultIndex -Results @($baseline.results) -Label 'Baseline'
|
|
$candidateIndex = Build-ResultIndex -Results @($candidate.results) -Label 'Candidate'
|
|
Assert-EqualValue 'result key set' @($baselineIndex.Keys | Sort-Object) @($candidateIndex.Keys | Sort-Object)
|
|
|
|
$budgets = $baseline.comparison_budgets
|
|
$metricDefinitions = @(
|
|
@('load_time_ms', 'load_time_ms_max_regression_percent', 'root'),
|
|
@('frame_ms_p95', 'frame_ms_p95_max_regression_percent', 'metrics'),
|
|
@('frame_ms_p99', 'frame_ms_p99_max_regression_percent', 'metrics'),
|
|
@('max_hitch_ms', 'max_hitch_ms_max_regression_percent', 'metrics'),
|
|
@('memory_static_bytes', 'memory_bytes_max_regression_percent', 'metrics'),
|
|
@('video_memory_bytes', 'memory_bytes_max_regression_percent', 'metrics')
|
|
)
|
|
|
|
foreach ($key in @($baselineIndex.Keys | Sort-Object)) {
|
|
if (-not $candidateIndex.ContainsKey($key)) {
|
|
continue
|
|
}
|
|
$baselineResult = $baselineIndex[$key]
|
|
$candidateResult = $candidateIndex[$key]
|
|
foreach ($definition in $metricDefinitions) {
|
|
$metricName = $definition[0]
|
|
$budgetName = $definition[1]
|
|
$location = $definition[2]
|
|
$baselineValue = if ($location -eq 'root') {
|
|
[double]$baselineResult.$metricName
|
|
} else {
|
|
[double]$baselineResult.metrics.$metricName
|
|
}
|
|
$candidateValue = if ($location -eq 'root') {
|
|
[double]$candidateResult.$metricName
|
|
} else {
|
|
[double]$candidateResult.metrics.$metricName
|
|
}
|
|
Compare-Metric `
|
|
-ResultKey $key `
|
|
-MetricName $metricName `
|
|
-BaselineValue $baselineValue `
|
|
-CandidateValue $candidateValue `
|
|
-AllowedRegressionPercent ([double]$budgets.$budgetName)
|
|
}
|
|
}
|
|
|
|
$summary = [pscustomobject]@{
|
|
schema_version = 1
|
|
baseline_revision = $baseline.revision
|
|
candidate_revision = $candidate.revision
|
|
baseline_created_utc = $baseline.created_utc
|
|
candidate_created_utc = $candidate.created_utc
|
|
baseline_samples = $baselineReports.Count
|
|
baseline_reports = @($BaselineReport)
|
|
candidate_samples = $candidateReports.Count
|
|
candidate_reports = @($CandidateReport)
|
|
result_pairs = $baselineIndex.Count
|
|
metric_comparisons = $comparisons.Count
|
|
passed = $failures.Count -eq 0
|
|
failures = @($failures)
|
|
comparisons = @($comparisons)
|
|
}
|
|
|
|
if ($OutputReport) {
|
|
$parent = Split-Path -Parent $OutputReport
|
|
if ($parent -and -not (Test-Path -LiteralPath $parent)) {
|
|
New-Item -ItemType Directory -Path $parent | Out-Null
|
|
}
|
|
$summary | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -LiteralPath $OutputReport
|
|
}
|
|
|
|
if ($failures.Count -gt 0) {
|
|
foreach ($failure in $failures) {
|
|
Write-Error "RENDER_PERFORMANCE: $failure" -ErrorAction Continue
|
|
}
|
|
Write-Host "RENDER_PERFORMANCE FAIL result_pairs=$($baselineIndex.Count) comparisons=$($comparisons.Count) failures=$($failures.Count)"
|
|
exit 1
|
|
}
|
|
|
|
Write-Host "RENDER_PERFORMANCE PASS result_pairs=$($baselineIndex.Count) comparisons=$($comparisons.Count) max_regression_percent=10"
|
|
exit 0
|