[CmdletBinding()] param( [Parameter(Mandatory = $true)] [string[]]$BaselineReport, [Parameter(Mandatory = $true)] [string[]]$CandidateReport, [string]$OutputReport ) $ErrorActionPreference = 'Stop' $failures = [System.Collections.Generic.List[string]]::new() $comparisons = [System.Collections.Generic.List[object]]::new() function Read-RenderReport { param( [Parameter(Mandatory = $true)] [string]$Path, [Parameter(Mandatory = $true)] [string]$Label ) if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "$Label report does not exist: $Path" } $report = Get-Content -Raw -Encoding UTF8 -LiteralPath $Path | ConvertFrom-Json if ($null -eq $report -or $null -eq $report.results) { throw "$Label report has no results array: $Path" } return $report } function Convert-ToStableJson { param([object]$Value) return $Value | ConvertTo-Json -Depth 20 -Compress } function Get-MedianValue { param([double[]]$Values) $ordered = @($Values | Sort-Object) if ($ordered.Count -eq 0) { throw 'Cannot calculate a median from an empty value set' } $middle = [math]::Floor($ordered.Count / 2) if ($ordered.Count % 2 -eq 1) { return [double]$ordered[$middle] } return ([double]$ordered[$middle - 1] + [double]$ordered[$middle]) / 2.0 } function Assert-EqualValue { param( [string]$Name, [object]$BaselineValue, [object]$CandidateValue ) $baselineJson = Convert-ToStableJson $BaselineValue $candidateJson = Convert-ToStableJson $CandidateValue if ($baselineJson -cne $candidateJson) { $failures.Add("$Name differs: baseline=$baselineJson candidate=$candidateJson") } } function Get-ResultKey { param([object]$Result) return "$($Result.name)|$($Result.pass)" } function Assert-CompatibleCacheInventory { param( [object]$BaselineInventory, [object]$CandidateInventory ) $baselineNames = @($BaselineInventory.PSObject.Properties.Name | Sort-Object) $candidateNames = @($CandidateInventory.PSObject.Properties.Name | Sort-Object) Assert-EqualValue 'cache inventory key set' $baselineNames $candidateNames foreach ($cacheName in $baselineNames) { $baselineRecord = $BaselineInventory.$cacheName $candidateRecord = $CandidateInventory.$cacheName if ($null -eq $candidateRecord) { continue } Assert-EqualValue "cache inventory presence for $cacheName" ` $baselineRecord.present ` $candidateRecord.present Assert-EqualValue "cache inventory file count for $cacheName" ` $baselineRecord.file_count ` $candidateRecord.file_count if ([bool]$baselineRecord.present -and [int64]$candidateRecord.file_count -le 0) { $failures.Add("cache inventory is empty for $cacheName") } } } function Build-ResultIndex { param( [object[]]$Results, [string]$Label ) $index = @{} foreach ($result in $Results) { $key = Get-ResultKey $result if ($index.ContainsKey($key)) { $failures.Add("$Label report contains duplicate result: $key") continue } $index[$key] = $result } return $index } function Merge-RenderReports { param( [object[]]$Reports, [string]$Label ) $merged = Convert-ToStableJson $Reports[0] | ConvertFrom-Json if ($Reports.Count -eq 1) { return $merged } $resultIndexes = @( foreach ($sample in $Reports) { Build-ResultIndex -Results @($sample.results) -Label "$Label sample" } ) $referenceResultKeys = @($resultIndexes[0].Keys | Sort-Object) for ($sampleNumber = 1; $sampleNumber -lt $Reports.Count; $sampleNumber++) { $sample = $Reports[$sampleNumber] Assert-EqualValue "$Label[$sampleNumber] schema_version" $merged.schema_version $sample.schema_version Assert-EqualValue "$Label[$sampleNumber] profile" $merged.profile $sample.profile Assert-EqualValue "$Label[$sampleNumber] cache_state" $merged.cache_state $sample.cache_state Assert-EqualValue "$Label[$sampleNumber] viewport" $merged.environment.viewport $sample.environment.viewport Assert-EqualValue "$Label[$sampleNumber] Godot version" $merged.environment.godot_version.string $sample.environment.godot_version.string Assert-EqualValue "$Label[$sampleNumber] rendering driver" $merged.environment.rendering_driver $sample.environment.rendering_driver Assert-EqualValue "$Label[$sampleNumber] rendering method" $merged.environment.rendering_method $sample.environment.rendering_method Assert-EqualValue "$Label[$sampleNumber] video adapter" $merged.environment.video_adapter $sample.environment.video_adapter Assert-EqualValue "$Label[$sampleNumber] CPU" $merged.environment.cpu $sample.environment.cpu Assert-EqualValue "$Label[$sampleNumber] cache contract" $merged.cache_contract $sample.cache_contract Assert-EqualValue "$Label[$sampleNumber] cache inventory" $merged.cache_inventory $sample.cache_inventory Assert-EqualValue ` "$Label[$sampleNumber] result key set" ` $referenceResultKeys ` @($resultIndexes[$sampleNumber].Keys | Sort-Object) } $metricDefinitions = @( @('load_time_ms', 'root'), @('frame_ms_p95', 'metrics'), @('frame_ms_p99', 'metrics'), @('max_hitch_ms', 'metrics'), @('memory_static_bytes', 'metrics'), @('video_memory_bytes', 'metrics') ) $mergedResultIndex = Build-ResultIndex -Results @($merged.results) -Label "Median $Label" foreach ($resultKey in $referenceResultKeys) { $mergedResult = $mergedResultIndex[$resultKey] foreach ($definition in $metricDefinitions) { $metricName = $definition[0] $location = $definition[1] $values = @( foreach ($sampleIndex in $resultIndexes) { if ($location -eq 'root') { [double]$sampleIndex[$resultKey].$metricName } else { [double]$sampleIndex[$resultKey].metrics.$metricName } } ) $medianValue = Get-MedianValue -Values $values if ($location -eq 'root') { $mergedResult.$metricName = $medianValue } else { $mergedResult.metrics.$metricName = $medianValue } } } return $merged } function Compare-Metric { param( [string]$ResultKey, [string]$MetricName, [double]$BaselineValue, [double]$CandidateValue, [double]$AllowedRegressionPercent ) $limit = if ($BaselineValue -eq 0.0) { 0.0 } else { $BaselineValue * (1.0 + $AllowedRegressionPercent / 100.0) } $regressionPercent = if ($BaselineValue -eq 0.0) { if ($CandidateValue -eq 0.0) { 0.0 } else { [double]::PositiveInfinity } } else { (($CandidateValue - $BaselineValue) / $BaselineValue) * 100.0 } $passed = $CandidateValue -le $limit $comparisons.Add([pscustomobject]@{ result = $ResultKey metric = $MetricName baseline = $BaselineValue candidate = $CandidateValue limit = $limit regression_percent = $regressionPercent passed = $passed }) if (-not $passed) { $failures.Add( ("{0} {1} regressed by {2:N3}%: baseline={3:N3} candidate={4:N3} limit={5:N3}" -f $ResultKey, $MetricName, $regressionPercent, $BaselineValue, $CandidateValue, $limit) ) } } $baselineReports = @( for ($baselineIndex = 0; $baselineIndex -lt $BaselineReport.Count; $baselineIndex++) { Read-RenderReport ` -Path $BaselineReport[$baselineIndex] ` -Label "Baseline[$baselineIndex]" } ) $candidateReports = @( for ($candidateIndex = 0; $candidateIndex -lt $CandidateReport.Count; $candidateIndex++) { Read-RenderReport ` -Path $CandidateReport[$candidateIndex] ` -Label "Candidate[$candidateIndex]" } ) $baseline = Merge-RenderReports -Reports $baselineReports -Label 'baseline' $candidate = Merge-RenderReports -Reports $candidateReports -Label 'candidate' Assert-EqualValue 'schema_version' $baseline.schema_version $candidate.schema_version Assert-EqualValue 'profile' $baseline.profile $candidate.profile Assert-EqualValue 'cache_state' $baseline.cache_state $candidate.cache_state Assert-EqualValue 'viewport' $baseline.environment.viewport $candidate.environment.viewport Assert-EqualValue 'Godot version' $baseline.environment.godot_version.string $candidate.environment.godot_version.string Assert-EqualValue 'rendering driver' $baseline.environment.rendering_driver $candidate.environment.rendering_driver Assert-EqualValue 'rendering method' $baseline.environment.rendering_method $candidate.environment.rendering_method Assert-EqualValue 'video adapter' $baseline.environment.video_adapter $candidate.environment.video_adapter Assert-EqualValue 'CPU' $baseline.environment.cpu $candidate.environment.cpu Assert-EqualValue 'cache contract' $baseline.cache_contract $candidate.cache_contract Assert-CompatibleCacheInventory $baseline.cache_inventory $candidate.cache_inventory $baselineIndex = Build-ResultIndex -Results @($baseline.results) -Label 'Baseline' $candidateIndex = Build-ResultIndex -Results @($candidate.results) -Label 'Candidate' Assert-EqualValue 'result key set' @($baselineIndex.Keys | Sort-Object) @($candidateIndex.Keys | Sort-Object) $budgets = $baseline.comparison_budgets $metricDefinitions = @( @('load_time_ms', 'load_time_ms_max_regression_percent', 'root'), @('frame_ms_p95', 'frame_ms_p95_max_regression_percent', 'metrics'), @('frame_ms_p99', 'frame_ms_p99_max_regression_percent', 'metrics'), @('max_hitch_ms', 'max_hitch_ms_max_regression_percent', 'metrics'), @('memory_static_bytes', 'memory_bytes_max_regression_percent', 'metrics'), @('video_memory_bytes', 'memory_bytes_max_regression_percent', 'metrics') ) foreach ($key in @($baselineIndex.Keys | Sort-Object)) { if (-not $candidateIndex.ContainsKey($key)) { continue } $baselineResult = $baselineIndex[$key] $candidateResult = $candidateIndex[$key] foreach ($definition in $metricDefinitions) { $metricName = $definition[0] $budgetName = $definition[1] $location = $definition[2] $baselineValue = if ($location -eq 'root') { [double]$baselineResult.$metricName } else { [double]$baselineResult.metrics.$metricName } $candidateValue = if ($location -eq 'root') { [double]$candidateResult.$metricName } else { [double]$candidateResult.metrics.$metricName } Compare-Metric ` -ResultKey $key ` -MetricName $metricName ` -BaselineValue $baselineValue ` -CandidateValue $candidateValue ` -AllowedRegressionPercent ([double]$budgets.$budgetName) } } $summary = [pscustomobject]@{ schema_version = 1 baseline_revision = $baseline.revision candidate_revision = $candidate.revision baseline_created_utc = $baseline.created_utc candidate_created_utc = $candidate.created_utc baseline_samples = $baselineReports.Count baseline_reports = @($BaselineReport) candidate_samples = $candidateReports.Count candidate_reports = @($CandidateReport) result_pairs = $baselineIndex.Count metric_comparisons = $comparisons.Count passed = $failures.Count -eq 0 failures = @($failures) comparisons = @($comparisons) } if ($OutputReport) { $parent = Split-Path -Parent $OutputReport if ($parent -and -not (Test-Path -LiteralPath $parent)) { New-Item -ItemType Directory -Path $parent | Out-Null } $summary | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -LiteralPath $OutputReport } if ($failures.Count -gt 0) { foreach ($failure in $failures) { Write-Error "RENDER_PERFORMANCE: $failure" -ErrorAction Continue } Write-Host "RENDER_PERFORMANCE FAIL result_pairs=$($baselineIndex.Count) comparisons=$($comparisons.Count) failures=$($failures.Count)" exit 1 } Write-Host "RENDER_PERFORMANCE PASS result_pairs=$($baselineIndex.Count) comparisons=$($comparisons.Count) max_regression_percent=10" exit 0