Complete M03 renderer closeout validation
This commit is contained in:
@@ -1,10 +1,10 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$BaselineReport,
|
||||
[string[]]$BaselineReport,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$CandidateReport,
|
||||
[string[]]$CandidateReport,
|
||||
|
||||
[string]$OutputReport
|
||||
)
|
||||
@@ -37,6 +37,20 @@ function Convert-ToStableJson {
|
||||
return $Value | ConvertTo-Json -Depth 20 -Compress
|
||||
}
|
||||
|
||||
function Get-MedianValue {
|
||||
param([double[]]$Values)
|
||||
|
||||
$ordered = @($Values | Sort-Object)
|
||||
if ($ordered.Count -eq 0) {
|
||||
throw 'Cannot calculate a median from an empty value set'
|
||||
}
|
||||
$middle = [math]::Floor($ordered.Count / 2)
|
||||
if ($ordered.Count % 2 -eq 1) {
|
||||
return [double]$ordered[$middle]
|
||||
}
|
||||
return ([double]$ordered[$middle - 1] + [double]$ordered[$middle]) / 2.0
|
||||
}
|
||||
|
||||
function Assert-EqualValue {
|
||||
param(
|
||||
[string]$Name,
|
||||
@@ -74,6 +88,9 @@ function Assert-CompatibleCacheInventory {
|
||||
Assert-EqualValue "cache inventory presence for $cacheName" `
|
||||
$baselineRecord.present `
|
||||
$candidateRecord.present
|
||||
Assert-EqualValue "cache inventory file count for $cacheName" `
|
||||
$baselineRecord.file_count `
|
||||
$candidateRecord.file_count
|
||||
if ([bool]$baselineRecord.present -and [int64]$candidateRecord.file_count -le 0) {
|
||||
$failures.Add("cache inventory is empty for $cacheName")
|
||||
}
|
||||
@@ -98,6 +115,75 @@ function Build-ResultIndex {
|
||||
return $index
|
||||
}
|
||||
|
||||
function Merge-RenderReports {
|
||||
param(
|
||||
[object[]]$Reports,
|
||||
[string]$Label
|
||||
)
|
||||
|
||||
$merged = Convert-ToStableJson $Reports[0] | ConvertFrom-Json
|
||||
if ($Reports.Count -eq 1) {
|
||||
return $merged
|
||||
}
|
||||
$resultIndexes = @(
|
||||
foreach ($sample in $Reports) {
|
||||
Build-ResultIndex -Results @($sample.results) -Label "$Label sample"
|
||||
}
|
||||
)
|
||||
$referenceResultKeys = @($resultIndexes[0].Keys | Sort-Object)
|
||||
for ($sampleNumber = 1; $sampleNumber -lt $Reports.Count; $sampleNumber++) {
|
||||
$sample = $Reports[$sampleNumber]
|
||||
Assert-EqualValue "$Label[$sampleNumber] schema_version" $merged.schema_version $sample.schema_version
|
||||
Assert-EqualValue "$Label[$sampleNumber] profile" $merged.profile $sample.profile
|
||||
Assert-EqualValue "$Label[$sampleNumber] cache_state" $merged.cache_state $sample.cache_state
|
||||
Assert-EqualValue "$Label[$sampleNumber] viewport" $merged.environment.viewport $sample.environment.viewport
|
||||
Assert-EqualValue "$Label[$sampleNumber] Godot version" $merged.environment.godot_version.string $sample.environment.godot_version.string
|
||||
Assert-EqualValue "$Label[$sampleNumber] rendering driver" $merged.environment.rendering_driver $sample.environment.rendering_driver
|
||||
Assert-EqualValue "$Label[$sampleNumber] rendering method" $merged.environment.rendering_method $sample.environment.rendering_method
|
||||
Assert-EqualValue "$Label[$sampleNumber] video adapter" $merged.environment.video_adapter $sample.environment.video_adapter
|
||||
Assert-EqualValue "$Label[$sampleNumber] CPU" $merged.environment.cpu $sample.environment.cpu
|
||||
Assert-EqualValue "$Label[$sampleNumber] cache contract" $merged.cache_contract $sample.cache_contract
|
||||
Assert-EqualValue "$Label[$sampleNumber] cache inventory" $merged.cache_inventory $sample.cache_inventory
|
||||
Assert-EqualValue `
|
||||
"$Label[$sampleNumber] result key set" `
|
||||
$referenceResultKeys `
|
||||
@($resultIndexes[$sampleNumber].Keys | Sort-Object)
|
||||
}
|
||||
|
||||
$metricDefinitions = @(
|
||||
@('load_time_ms', 'root'),
|
||||
@('frame_ms_p95', 'metrics'),
|
||||
@('frame_ms_p99', 'metrics'),
|
||||
@('max_hitch_ms', 'metrics'),
|
||||
@('memory_static_bytes', 'metrics'),
|
||||
@('video_memory_bytes', 'metrics')
|
||||
)
|
||||
$mergedResultIndex = Build-ResultIndex -Results @($merged.results) -Label "Median $Label"
|
||||
foreach ($resultKey in $referenceResultKeys) {
|
||||
$mergedResult = $mergedResultIndex[$resultKey]
|
||||
foreach ($definition in $metricDefinitions) {
|
||||
$metricName = $definition[0]
|
||||
$location = $definition[1]
|
||||
$values = @(
|
||||
foreach ($sampleIndex in $resultIndexes) {
|
||||
if ($location -eq 'root') {
|
||||
[double]$sampleIndex[$resultKey].$metricName
|
||||
} else {
|
||||
[double]$sampleIndex[$resultKey].metrics.$metricName
|
||||
}
|
||||
}
|
||||
)
|
||||
$medianValue = Get-MedianValue -Values $values
|
||||
if ($location -eq 'root') {
|
||||
$mergedResult.$metricName = $medianValue
|
||||
} else {
|
||||
$mergedResult.metrics.$metricName = $medianValue
|
||||
}
|
||||
}
|
||||
}
|
||||
return $merged
|
||||
}
|
||||
|
||||
function Compare-Metric {
|
||||
param(
|
||||
[string]$ResultKey,
|
||||
@@ -140,8 +226,22 @@ function Compare-Metric {
|
||||
}
|
||||
}
|
||||
|
||||
$baseline = Read-RenderReport -Path $BaselineReport -Label 'Baseline'
|
||||
$candidate = Read-RenderReport -Path $CandidateReport -Label 'Candidate'
|
||||
$baselineReports = @(
|
||||
for ($baselineIndex = 0; $baselineIndex -lt $BaselineReport.Count; $baselineIndex++) {
|
||||
Read-RenderReport `
|
||||
-Path $BaselineReport[$baselineIndex] `
|
||||
-Label "Baseline[$baselineIndex]"
|
||||
}
|
||||
)
|
||||
$candidateReports = @(
|
||||
for ($candidateIndex = 0; $candidateIndex -lt $CandidateReport.Count; $candidateIndex++) {
|
||||
Read-RenderReport `
|
||||
-Path $CandidateReport[$candidateIndex] `
|
||||
-Label "Candidate[$candidateIndex]"
|
||||
}
|
||||
)
|
||||
$baseline = Merge-RenderReports -Reports $baselineReports -Label 'baseline'
|
||||
$candidate = Merge-RenderReports -Reports $candidateReports -Label 'candidate'
|
||||
|
||||
Assert-EqualValue 'schema_version' $baseline.schema_version $candidate.schema_version
|
||||
Assert-EqualValue 'profile' $baseline.profile $candidate.profile
|
||||
@@ -204,6 +304,10 @@ $summary = [pscustomobject]@{
|
||||
candidate_revision = $candidate.revision
|
||||
baseline_created_utc = $baseline.created_utc
|
||||
candidate_created_utc = $candidate.created_utc
|
||||
baseline_samples = $baselineReports.Count
|
||||
baseline_reports = @($BaselineReport)
|
||||
candidate_samples = $candidateReports.Count
|
||||
candidate_reports = @($CandidateReport)
|
||||
result_pairs = $baselineIndex.Count
|
||||
metric_comparisons = $comparisons.Count
|
||||
passed = $failures.Count -eq 0
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RepeatedSampleComparison,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$LongWindowComparison,
|
||||
|
||||
[string]$OutputReport
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Read-ComparisonReport {
|
||||
param([string]$Path, [string]$Label)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
throw "$Label comparison does not exist: $Path"
|
||||
}
|
||||
$report = Get-Content -Raw -Encoding UTF8 -LiteralPath $Path | ConvertFrom-Json
|
||||
if ($null -eq $report -or $null -eq $report.comparisons) {
|
||||
throw "$Label comparison has no comparisons array: $Path"
|
||||
}
|
||||
return $report
|
||||
}
|
||||
|
||||
function Build-ComparisonIndex {
|
||||
param([object[]]$Comparisons, [string]$Label)
|
||||
|
||||
$index = @{}
|
||||
foreach ($comparison in $Comparisons) {
|
||||
$key = "$($comparison.result)|$($comparison.metric)"
|
||||
if ($index.ContainsKey($key)) {
|
||||
throw "$Label comparison contains duplicate metric: $key"
|
||||
}
|
||||
$index[$key] = $comparison
|
||||
}
|
||||
return $index
|
||||
}
|
||||
|
||||
$repeatedReport = Read-ComparisonReport `
|
||||
-Path $RepeatedSampleComparison `
|
||||
-Label 'Repeated-sample'
|
||||
$longWindowReport = Read-ComparisonReport `
|
||||
-Path $LongWindowComparison `
|
||||
-Label 'Long-window'
|
||||
$repeatedIndex = Build-ComparisonIndex `
|
||||
-Comparisons @($repeatedReport.comparisons) `
|
||||
-Label 'Repeated-sample'
|
||||
$longWindowIndex = Build-ComparisonIndex `
|
||||
-Comparisons @($longWindowReport.comparisons) `
|
||||
-Label 'Long-window'
|
||||
$repeatedKeys = @($repeatedIndex.Keys | Sort-Object)
|
||||
$longWindowKeys = @($longWindowIndex.Keys | Sort-Object)
|
||||
if (($repeatedKeys | ConvertTo-Json -Compress) -cne ($longWindowKeys | ConvertTo-Json -Compress)) {
|
||||
throw 'Comparison metric inventories differ between repeated and long-window protocols'
|
||||
}
|
||||
|
||||
$repeatableRegressions = [System.Collections.Generic.List[object]]::new()
|
||||
$nonRepeatableRegressions = [System.Collections.Generic.List[object]]::new()
|
||||
foreach ($key in $repeatedKeys) {
|
||||
$repeatedMetricComparison = $repeatedIndex[$key]
|
||||
$longWindowMetricComparison = $longWindowIndex[$key]
|
||||
$repeatedFailed = ([bool]$repeatedMetricComparison.passed) -eq $false
|
||||
$longWindowFailed = ([bool]$longWindowMetricComparison.passed) -eq $false
|
||||
if ($repeatedFailed -and $longWindowFailed) {
|
||||
$repeatableRegressions.Add([pscustomobject]@{
|
||||
key = $key
|
||||
repeated_sample = $repeatedMetricComparison
|
||||
long_window = $longWindowMetricComparison
|
||||
})
|
||||
} elseif ($repeatedFailed -or $longWindowFailed) {
|
||||
$nonRepeatableRegressions.Add([pscustomobject]@{
|
||||
key = $key
|
||||
failed_protocol = if ($repeatedFailed) { 'repeated_sample' } else { 'long_window' }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
$summary = [pscustomobject]@{
|
||||
schema_version = 1
|
||||
repeated_sample_comparison = $RepeatedSampleComparison
|
||||
long_window_comparison = $LongWindowComparison
|
||||
metric_comparisons = $repeatedKeys.Count
|
||||
passed = $repeatableRegressions.Count -eq 0
|
||||
repeatable_regressions = @($repeatableRegressions)
|
||||
non_repeatable_regressions = @($nonRepeatableRegressions)
|
||||
}
|
||||
if ($OutputReport) {
|
||||
$parent = Split-Path -Parent $OutputReport
|
||||
if ($parent -and -not (Test-Path -LiteralPath $parent)) {
|
||||
New-Item -ItemType Directory -Path $parent | Out-Null
|
||||
}
|
||||
$summary | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -LiteralPath $OutputReport
|
||||
}
|
||||
|
||||
if ($repeatableRegressions.Count -gt 0) {
|
||||
foreach ($regression in $repeatableRegressions) {
|
||||
Write-Error "RENDER_PERFORMANCE_STABILITY: repeatable regression $($regression.key)" -ErrorAction Continue
|
||||
}
|
||||
Write-Host "RENDER_PERFORMANCE_STABILITY FAIL repeated=$($repeatableRegressions.Count) metrics=$($repeatedKeys.Count)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "RENDER_PERFORMANCE_STABILITY PASS metrics=$($repeatedKeys.Count) non_repeatable=$($nonRepeatableRegressions.Count)"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user