Complete M03 renderer closeout validation

2026-08-02 15:28:10 +04:00
parent d9d20cb53f
commit 3bf1c2c657
13 changed files with 588 additions and 36 deletions
@@ -1,10 +1,10 @@
 [CmdletBinding()]
 param(
    [Parameter(Mandatory = $true)]
-    [string]$BaselineReport,
+    [string[]]$BaselineReport,

    [Parameter(Mandatory = $true)]
-    [string]$CandidateReport,
+    [string[]]$CandidateReport,

    [string]$OutputReport
 )
@@ -37,6 +37,20 @@ function Convert-ToStableJson {
    return $Value | ConvertTo-Json -Depth 20 -Compress
 }

+function Get-MedianValue {
+    param([double[]]$Values)
+
+    $ordered = @($Values | Sort-Object)
+    if ($ordered.Count -eq 0) {
+        throw 'Cannot calculate a median from an empty value set'
+    }
+    $middle = [math]::Floor($ordered.Count / 2)
+    if ($ordered.Count % 2 -eq 1) {
+        return [double]$ordered[$middle]
+    }
+    return ([double]$ordered[$middle - 1] + [double]$ordered[$middle]) / 2.0
+}
+
 function Assert-EqualValue {
    param(
        [string]$Name,
@@ -74,6 +88,9 @@ function Assert-CompatibleCacheInventory {
        Assert-EqualValue "cache inventory presence for $cacheName" `
            $baselineRecord.present `
            $candidateRecord.present
+        Assert-EqualValue "cache inventory file count for $cacheName" `
+            $baselineRecord.file_count `
+            $candidateRecord.file_count
        if ([bool]$baselineRecord.present -and [int64]$candidateRecord.file_count -le 0) {
            $failures.Add("cache inventory is empty for $cacheName")
        }
@@ -98,6 +115,75 @@ function Build-ResultIndex {
    return $index
 }

+function Merge-RenderReports {
+    param(
+        [object[]]$Reports,
+        [string]$Label
+    )
+
+    $merged = Convert-ToStableJson $Reports[0] | ConvertFrom-Json
+    if ($Reports.Count -eq 1) {
+        return $merged
+    }
+    $resultIndexes = @(
+        foreach ($sample in $Reports) {
+            Build-ResultIndex -Results @($sample.results) -Label "$Label sample"
+        }
+    )
+    $referenceResultKeys = @($resultIndexes[0].Keys | Sort-Object)
+    for ($sampleNumber = 1; $sampleNumber -lt $Reports.Count; $sampleNumber++) {
+        $sample = $Reports[$sampleNumber]
+        Assert-EqualValue "$Label[$sampleNumber] schema_version" $merged.schema_version $sample.schema_version
+        Assert-EqualValue "$Label[$sampleNumber] profile" $merged.profile $sample.profile
+        Assert-EqualValue "$Label[$sampleNumber] cache_state" $merged.cache_state $sample.cache_state
+        Assert-EqualValue "$Label[$sampleNumber] viewport" $merged.environment.viewport $sample.environment.viewport
+        Assert-EqualValue "$Label[$sampleNumber] Godot version" $merged.environment.godot_version.string $sample.environment.godot_version.string
+        Assert-EqualValue "$Label[$sampleNumber] rendering driver" $merged.environment.rendering_driver $sample.environment.rendering_driver
+        Assert-EqualValue "$Label[$sampleNumber] rendering method" $merged.environment.rendering_method $sample.environment.rendering_method
+        Assert-EqualValue "$Label[$sampleNumber] video adapter" $merged.environment.video_adapter $sample.environment.video_adapter
+        Assert-EqualValue "$Label[$sampleNumber] CPU" $merged.environment.cpu $sample.environment.cpu
+        Assert-EqualValue "$Label[$sampleNumber] cache contract" $merged.cache_contract $sample.cache_contract
+        Assert-EqualValue "$Label[$sampleNumber] cache inventory" $merged.cache_inventory $sample.cache_inventory
+        Assert-EqualValue `
+            "$Label[$sampleNumber] result key set" `
+            $referenceResultKeys `
+            @($resultIndexes[$sampleNumber].Keys | Sort-Object)
+    }
+
+    $metricDefinitions = @(
+        @('load_time_ms', 'root'),
+        @('frame_ms_p95', 'metrics'),
+        @('frame_ms_p99', 'metrics'),
+        @('max_hitch_ms', 'metrics'),
+        @('memory_static_bytes', 'metrics'),
+        @('video_memory_bytes', 'metrics')
+    )
+    $mergedResultIndex = Build-ResultIndex -Results @($merged.results) -Label "Median $Label"
+    foreach ($resultKey in $referenceResultKeys) {
+        $mergedResult = $mergedResultIndex[$resultKey]
+        foreach ($definition in $metricDefinitions) {
+            $metricName = $definition[0]
+            $location = $definition[1]
+            $values = @(
+                foreach ($sampleIndex in $resultIndexes) {
+                    if ($location -eq 'root') {
+                        [double]$sampleIndex[$resultKey].$metricName
+                    } else {
+                        [double]$sampleIndex[$resultKey].metrics.$metricName
+                    }
+                }
+            )
+            $medianValue = Get-MedianValue -Values $values
+            if ($location -eq 'root') {
+                $mergedResult.$metricName = $medianValue
+            } else {
+                $mergedResult.metrics.$metricName = $medianValue
+            }
+        }
+    }
+    return $merged
+}
+
 function Compare-Metric {
    param(
        [string]$ResultKey,
@@ -140,8 +226,22 @@ function Compare-Metric {
    }
 }

-$baseline = Read-RenderReport -Path $BaselineReport -Label 'Baseline'
-$candidate = Read-RenderReport -Path $CandidateReport -Label 'Candidate'
+$baselineReports = @(
+    for ($baselineIndex = 0; $baselineIndex -lt $BaselineReport.Count; $baselineIndex++) {
+        Read-RenderReport `
+            -Path $BaselineReport[$baselineIndex] `
+            -Label "Baseline[$baselineIndex]"
+    }
+)
+$candidateReports = @(
+    for ($candidateIndex = 0; $candidateIndex -lt $CandidateReport.Count; $candidateIndex++) {
+        Read-RenderReport `
+            -Path $CandidateReport[$candidateIndex] `
+            -Label "Candidate[$candidateIndex]"
+    }
+)
+$baseline = Merge-RenderReports -Reports $baselineReports -Label 'baseline'
+$candidate = Merge-RenderReports -Reports $candidateReports -Label 'candidate'

 Assert-EqualValue 'schema_version' $baseline.schema_version $candidate.schema_version
 Assert-EqualValue 'profile' $baseline.profile $candidate.profile
@@ -204,6 +304,10 @@ $summary = [pscustomobject]@{
    candidate_revision = $candidate.revision
    baseline_created_utc = $baseline.created_utc
    candidate_created_utc = $candidate.created_utc
+    baseline_samples = $baselineReports.Count
+    baseline_reports = @($BaselineReport)
+    candidate_samples = $candidateReports.Count
+    candidate_reports = @($CandidateReport)
    result_pairs = $baselineIndex.Count
    metric_comparisons = $comparisons.Count
    passed = $failures.Count -eq 0