qar: add M03 renderer closeout gates
This commit is contained in:
@@ -176,14 +176,14 @@ func _bake_glb_animation_cache(
|
||||
if not force and FileAccess.file_exists(abs_out_glb):
|
||||
return true
|
||||
var abs_converter := ProjectSettings.globalize_path(converter)
|
||||
var abs_output := ProjectSettings.globalize_path(output_dir)
|
||||
var converter_output_directory := abs_out_glb.get_base_dir()
|
||||
if not FileAccess.file_exists(abs_converter):
|
||||
push_warning("M2 GLB converter not found: %s" % converter)
|
||||
return false
|
||||
var stdout := []
|
||||
var exit_code := OS.execute(
|
||||
python_exe,
|
||||
[abs_converter, abs_m2, abs_output],
|
||||
[abs_converter, abs_m2, converter_output_directory],
|
||||
stdout,
|
||||
true,
|
||||
false)
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
extends SceneTree
|
||||
|
||||
## M03 closeout gate for cache versions and worker/main-thread render boundaries.
|
||||
|
||||
const LOADER_PATH := "res://src/scenes/streaming/streaming_world_loader.gd"
|
||||
const MANIFEST_PATH := "res://src/tools/render_baseline_manifest.json"
|
||||
const M2_CACHE_BAKER_PATH := "res://src/tools/bake_m2_cache.gd"
|
||||
const WORKER_FUNCTIONS: Array[String] = [
|
||||
"_load_tile_task",
|
||||
"_load_terrain_splat_task",
|
||||
"_load_tile_water_task",
|
||||
"_group_tile_m2_task",
|
||||
]
|
||||
const WORKER_FORBIDDEN_TOKENS: Array[String] = [
|
||||
"RenderingServer",
|
||||
".add_child(",
|
||||
".queue_free(",
|
||||
".free(",
|
||||
"Node3D.new(",
|
||||
"MeshInstance3D.new(",
|
||||
"MultiMeshInstance3D.new(",
|
||||
"ArrayMesh.new(",
|
||||
"build_tile_water_scene(",
|
||||
"build_tile_coarse_render_payload(",
|
||||
]
|
||||
const MAIN_THREAD_FRAME_STEPS: Array[String] = [
|
||||
"_drain_tile_load_results()",
|
||||
"_drain_terrain_upgrade_results()",
|
||||
"_drain_terrain_control_splat_cache_results()",
|
||||
"_drain_terrain_splat_cache_results()",
|
||||
"_drain_terrain_splat_results()",
|
||||
"_process_water_load_queue()",
|
||||
"_drain_water_load_results()",
|
||||
"_process_queues()",
|
||||
"_drain_m2_group_results()",
|
||||
"_drain_m2_animation_loads()",
|
||||
"_drain_m2_mesh_loads()",
|
||||
"_process_m2_build_jobs()",
|
||||
"_process_wmo_build_jobs()",
|
||||
"_process_wmo_render_build_jobs()",
|
||||
"_process_detail_asset_queue()",
|
||||
]
|
||||
|
||||
|
||||
func _initialize() -> void:
|
||||
var failures: Array[String] = []
|
||||
var loader_source := _read_text(LOADER_PATH, failures)
|
||||
var process_source := _function_source(loader_source, "_process", failures)
|
||||
_verify_frame_entrypoint(process_source, failures)
|
||||
_verify_worker_boundaries(loader_source, failures)
|
||||
_verify_rendering_server_boundary(loader_source, failures)
|
||||
_verify_cache_contract(loader_source, failures)
|
||||
_verify_m2_glb_cache_output_contract(failures)
|
||||
|
||||
if not failures.is_empty():
|
||||
for failure in failures:
|
||||
push_error("RENDERER_CLOSEOUT_CONTRACTS: %s" % failure)
|
||||
quit(1)
|
||||
return
|
||||
|
||||
print("RENDERER_CLOSEOUT_CONTRACTS PASS workers=%d frame_steps=%d cache_versions=7" % [
|
||||
WORKER_FUNCTIONS.size(),
|
||||
MAIN_THREAD_FRAME_STEPS.size(),
|
||||
])
|
||||
quit(0)
|
||||
|
||||
|
||||
func _verify_frame_entrypoint(process_source: String, failures: Array[String]) -> void:
|
||||
var scheduler_index := process_source.find("_render_budget_scheduler.begin_frame(")
|
||||
if scheduler_index < 0:
|
||||
failures.append("_process does not begin a render-budget frame")
|
||||
return
|
||||
for step in MAIN_THREAD_FRAME_STEPS:
|
||||
var step_index := process_source.find(step)
|
||||
if step_index < 0:
|
||||
failures.append("_process is missing main-thread step %s" % step)
|
||||
elif step_index < scheduler_index:
|
||||
failures.append("%s runs before render-budget frame admission" % step)
|
||||
|
||||
|
||||
func _verify_worker_boundaries(loader_source: String, failures: Array[String]) -> void:
|
||||
for function_name in WORKER_FUNCTIONS:
|
||||
var function_source := _function_source(loader_source, function_name, failures)
|
||||
for forbidden_token in WORKER_FORBIDDEN_TOKENS:
|
||||
if function_source.contains(forbidden_token):
|
||||
failures.append("worker %s contains main-thread token %s" % [
|
||||
function_name,
|
||||
forbidden_token,
|
||||
])
|
||||
|
||||
|
||||
func _verify_rendering_server_boundary(loader_source: String, failures: Array[String]) -> void:
|
||||
var remaining_source := loader_source
|
||||
for function_name in ["_create_render_instance", "_free_render_instance"]:
|
||||
var function_source := _function_source(loader_source, function_name, failures)
|
||||
remaining_source = remaining_source.replace(function_source, "")
|
||||
if remaining_source.contains("RenderingServer.instance_create("):
|
||||
failures.append("RenderingServer instance creation escaped the owned adapter")
|
||||
if remaining_source.contains("RenderingServer.free_rid("):
|
||||
failures.append("RenderingServer RID release escaped the owned adapter")
|
||||
|
||||
|
||||
func _verify_cache_contract(loader_source: String, failures: Array[String]) -> void:
|
||||
var manifest_source := _read_text(MANIFEST_PATH, failures)
|
||||
var parsed_manifest = JSON.parse_string(manifest_source)
|
||||
if not (parsed_manifest is Dictionary):
|
||||
failures.append("baseline manifest is not a Dictionary")
|
||||
return
|
||||
var cache_contract: Dictionary = parsed_manifest.get("cache_contract", {})
|
||||
var expected_versions := {
|
||||
"baked_terrain": 5,
|
||||
"streaming_terrain": 2,
|
||||
"terrain_splat": 1,
|
||||
"terrain_control_splat": 3,
|
||||
"wmo_streaming": 2,
|
||||
"wmo_builder": 2,
|
||||
"m2_material": 2,
|
||||
}
|
||||
if cache_contract.size() != expected_versions.size():
|
||||
failures.append("cache contract key count changed: expected=%d actual=%d" % [
|
||||
expected_versions.size(),
|
||||
cache_contract.size(),
|
||||
])
|
||||
for cache_name in expected_versions:
|
||||
if not cache_contract.has(cache_name):
|
||||
failures.append("cache contract is missing %s" % cache_name)
|
||||
continue
|
||||
var cache_record: Dictionary = cache_contract[cache_name]
|
||||
if int(cache_record.get("version", -1)) != int(expected_versions[cache_name]):
|
||||
failures.append("cache version changed for %s" % cache_name)
|
||||
|
||||
var required_loader_declarations := [
|
||||
"const REQUIRED_BAKED_TILE_FORMAT_VERSION := 5",
|
||||
"const REQUIRED_SPLAT_TILE_FORMAT_VERSION := 1",
|
||||
"const REQUIRED_CONTROL_SPLAT_TILE_FORMAT_VERSION := 3",
|
||||
"STREAMING_TILE_SCRIPT.FORMAT_VERSION",
|
||||
"WMO_STREAMING_SCRIPT.FORMAT_VERSION",
|
||||
]
|
||||
for declaration in required_loader_declarations:
|
||||
if not loader_source.contains(declaration):
|
||||
failures.append("loader cache-version boundary is missing %s" % declaration)
|
||||
|
||||
|
||||
func _verify_m2_glb_cache_output_contract(failures: Array[String]) -> void:
|
||||
var baker_source := _read_text(M2_CACHE_BAKER_PATH, failures)
|
||||
if not baker_source.contains(
|
||||
"var converter_output_directory := abs_out_glb.get_base_dir()"):
|
||||
failures.append("M2 GLB converter output is not derived from the nested cache path")
|
||||
if not baker_source.contains(
|
||||
"[abs_converter, abs_m2, converter_output_directory]"):
|
||||
failures.append("M2 GLB converter does not receive the nested output directory")
|
||||
|
||||
|
||||
func _function_source(source: String, function_name: String, failures: Array[String]) -> String:
|
||||
var signature := "func %s(" % function_name
|
||||
var start_index := source.find(signature)
|
||||
if start_index < 0:
|
||||
failures.append("missing function %s" % function_name)
|
||||
return ""
|
||||
var next_function_index := source.find("\nfunc ", start_index + signature.length())
|
||||
if next_function_index < 0:
|
||||
return source.substr(start_index)
|
||||
return source.substr(start_index, next_function_index - start_index)
|
||||
|
||||
|
||||
func _read_text(path: String, failures: Array[String]) -> String:
|
||||
var file := FileAccess.open(path, FileAccess.READ)
|
||||
if file == null:
|
||||
failures.append("cannot read %s" % path)
|
||||
return ""
|
||||
var content := file.get_as_text()
|
||||
file.close()
|
||||
return content
|
||||
@@ -0,0 +1 @@
|
||||
uid://nqj0mu6r8omf
|
||||
@@ -0,0 +1,231 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$BaselineReport,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$CandidateReport,
|
||||
|
||||
[string]$OutputReport
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$failures = [System.Collections.Generic.List[string]]::new()
|
||||
$comparisons = [System.Collections.Generic.List[object]]::new()
|
||||
|
||||
function Read-RenderReport {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Path,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Label
|
||||
)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
throw "$Label report does not exist: $Path"
|
||||
}
|
||||
$report = Get-Content -Raw -Encoding UTF8 -LiteralPath $Path | ConvertFrom-Json
|
||||
if ($null -eq $report -or $null -eq $report.results) {
|
||||
throw "$Label report has no results array: $Path"
|
||||
}
|
||||
return $report
|
||||
}
|
||||
|
||||
function Convert-ToStableJson {
|
||||
param([object]$Value)
|
||||
return $Value | ConvertTo-Json -Depth 20 -Compress
|
||||
}
|
||||
|
||||
function Assert-EqualValue {
|
||||
param(
|
||||
[string]$Name,
|
||||
[object]$BaselineValue,
|
||||
[object]$CandidateValue
|
||||
)
|
||||
|
||||
$baselineJson = Convert-ToStableJson $BaselineValue
|
||||
$candidateJson = Convert-ToStableJson $CandidateValue
|
||||
if ($baselineJson -cne $candidateJson) {
|
||||
$failures.Add("$Name differs: baseline=$baselineJson candidate=$candidateJson")
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ResultKey {
|
||||
param([object]$Result)
|
||||
return "$($Result.name)|$($Result.pass)"
|
||||
}
|
||||
|
||||
function Assert-CompatibleCacheInventory {
|
||||
param(
|
||||
[object]$BaselineInventory,
|
||||
[object]$CandidateInventory
|
||||
)
|
||||
|
||||
$baselineNames = @($BaselineInventory.PSObject.Properties.Name | Sort-Object)
|
||||
$candidateNames = @($CandidateInventory.PSObject.Properties.Name | Sort-Object)
|
||||
Assert-EqualValue 'cache inventory key set' $baselineNames $candidateNames
|
||||
foreach ($cacheName in $baselineNames) {
|
||||
$baselineRecord = $BaselineInventory.$cacheName
|
||||
$candidateRecord = $CandidateInventory.$cacheName
|
||||
if ($null -eq $candidateRecord) {
|
||||
continue
|
||||
}
|
||||
Assert-EqualValue "cache inventory presence for $cacheName" `
|
||||
$baselineRecord.present `
|
||||
$candidateRecord.present
|
||||
if ([bool]$baselineRecord.present -and [int64]$candidateRecord.file_count -le 0) {
|
||||
$failures.Add("cache inventory is empty for $cacheName")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Build-ResultIndex {
|
||||
param(
|
||||
[object[]]$Results,
|
||||
[string]$Label
|
||||
)
|
||||
|
||||
$index = @{}
|
||||
foreach ($result in $Results) {
|
||||
$key = Get-ResultKey $result
|
||||
if ($index.ContainsKey($key)) {
|
||||
$failures.Add("$Label report contains duplicate result: $key")
|
||||
continue
|
||||
}
|
||||
$index[$key] = $result
|
||||
}
|
||||
return $index
|
||||
}
|
||||
|
||||
function Compare-Metric {
|
||||
param(
|
||||
[string]$ResultKey,
|
||||
[string]$MetricName,
|
||||
[double]$BaselineValue,
|
||||
[double]$CandidateValue,
|
||||
[double]$AllowedRegressionPercent
|
||||
)
|
||||
|
||||
$limit = if ($BaselineValue -eq 0.0) {
|
||||
0.0
|
||||
} else {
|
||||
$BaselineValue * (1.0 + $AllowedRegressionPercent / 100.0)
|
||||
}
|
||||
$regressionPercent = if ($BaselineValue -eq 0.0) {
|
||||
if ($CandidateValue -eq 0.0) { 0.0 } else { [double]::PositiveInfinity }
|
||||
} else {
|
||||
(($CandidateValue - $BaselineValue) / $BaselineValue) * 100.0
|
||||
}
|
||||
$passed = $CandidateValue -le $limit
|
||||
$comparisons.Add([pscustomobject]@{
|
||||
result = $ResultKey
|
||||
metric = $MetricName
|
||||
baseline = $BaselineValue
|
||||
candidate = $CandidateValue
|
||||
limit = $limit
|
||||
regression_percent = $regressionPercent
|
||||
passed = $passed
|
||||
})
|
||||
if (-not $passed) {
|
||||
$failures.Add(
|
||||
("{0} {1} regressed by {2:N3}%: baseline={3:N3} candidate={4:N3} limit={5:N3}" -f
|
||||
$ResultKey,
|
||||
$MetricName,
|
||||
$regressionPercent,
|
||||
$BaselineValue,
|
||||
$CandidateValue,
|
||||
$limit)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
$baseline = Read-RenderReport -Path $BaselineReport -Label 'Baseline'
|
||||
$candidate = Read-RenderReport -Path $CandidateReport -Label 'Candidate'
|
||||
|
||||
Assert-EqualValue 'schema_version' $baseline.schema_version $candidate.schema_version
|
||||
Assert-EqualValue 'profile' $baseline.profile $candidate.profile
|
||||
Assert-EqualValue 'cache_state' $baseline.cache_state $candidate.cache_state
|
||||
Assert-EqualValue 'viewport' $baseline.environment.viewport $candidate.environment.viewport
|
||||
Assert-EqualValue 'Godot version' $baseline.environment.godot_version.string $candidate.environment.godot_version.string
|
||||
Assert-EqualValue 'rendering driver' $baseline.environment.rendering_driver $candidate.environment.rendering_driver
|
||||
Assert-EqualValue 'rendering method' $baseline.environment.rendering_method $candidate.environment.rendering_method
|
||||
Assert-EqualValue 'video adapter' $baseline.environment.video_adapter $candidate.environment.video_adapter
|
||||
Assert-EqualValue 'CPU' $baseline.environment.cpu $candidate.environment.cpu
|
||||
Assert-EqualValue 'cache contract' $baseline.cache_contract $candidate.cache_contract
|
||||
Assert-CompatibleCacheInventory $baseline.cache_inventory $candidate.cache_inventory
|
||||
|
||||
$baselineIndex = Build-ResultIndex -Results @($baseline.results) -Label 'Baseline'
|
||||
$candidateIndex = Build-ResultIndex -Results @($candidate.results) -Label 'Candidate'
|
||||
Assert-EqualValue 'result key set' @($baselineIndex.Keys | Sort-Object) @($candidateIndex.Keys | Sort-Object)
|
||||
|
||||
$budgets = $baseline.comparison_budgets
|
||||
$metricDefinitions = @(
|
||||
@('load_time_ms', 'load_time_ms_max_regression_percent', 'root'),
|
||||
@('frame_ms_p95', 'frame_ms_p95_max_regression_percent', 'metrics'),
|
||||
@('frame_ms_p99', 'frame_ms_p99_max_regression_percent', 'metrics'),
|
||||
@('max_hitch_ms', 'max_hitch_ms_max_regression_percent', 'metrics'),
|
||||
@('memory_static_bytes', 'memory_bytes_max_regression_percent', 'metrics'),
|
||||
@('video_memory_bytes', 'memory_bytes_max_regression_percent', 'metrics')
|
||||
)
|
||||
|
||||
foreach ($key in @($baselineIndex.Keys | Sort-Object)) {
|
||||
if (-not $candidateIndex.ContainsKey($key)) {
|
||||
continue
|
||||
}
|
||||
$baselineResult = $baselineIndex[$key]
|
||||
$candidateResult = $candidateIndex[$key]
|
||||
foreach ($definition in $metricDefinitions) {
|
||||
$metricName = $definition[0]
|
||||
$budgetName = $definition[1]
|
||||
$location = $definition[2]
|
||||
$baselineValue = if ($location -eq 'root') {
|
||||
[double]$baselineResult.$metricName
|
||||
} else {
|
||||
[double]$baselineResult.metrics.$metricName
|
||||
}
|
||||
$candidateValue = if ($location -eq 'root') {
|
||||
[double]$candidateResult.$metricName
|
||||
} else {
|
||||
[double]$candidateResult.metrics.$metricName
|
||||
}
|
||||
Compare-Metric `
|
||||
-ResultKey $key `
|
||||
-MetricName $metricName `
|
||||
-BaselineValue $baselineValue `
|
||||
-CandidateValue $candidateValue `
|
||||
-AllowedRegressionPercent ([double]$budgets.$budgetName)
|
||||
}
|
||||
}
|
||||
|
||||
$summary = [pscustomobject]@{
|
||||
schema_version = 1
|
||||
baseline_revision = $baseline.revision
|
||||
candidate_revision = $candidate.revision
|
||||
baseline_created_utc = $baseline.created_utc
|
||||
candidate_created_utc = $candidate.created_utc
|
||||
result_pairs = $baselineIndex.Count
|
||||
metric_comparisons = $comparisons.Count
|
||||
passed = $failures.Count -eq 0
|
||||
failures = @($failures)
|
||||
comparisons = @($comparisons)
|
||||
}
|
||||
|
||||
if ($OutputReport) {
|
||||
$parent = Split-Path -Parent $OutputReport
|
||||
if ($parent -and -not (Test-Path -LiteralPath $parent)) {
|
||||
New-Item -ItemType Directory -Path $parent | Out-Null
|
||||
}
|
||||
$summary | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -LiteralPath $OutputReport
|
||||
}
|
||||
|
||||
if ($failures.Count -gt 0) {
|
||||
foreach ($failure in $failures) {
|
||||
Write-Error "RENDER_PERFORMANCE: $failure" -ErrorAction Continue
|
||||
}
|
||||
Write-Host "RENDER_PERFORMANCE FAIL result_pairs=$($baselineIndex.Count) comparisons=$($comparisons.Count) failures=$($failures.Count)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "RENDER_PERFORMANCE PASS result_pairs=$($baselineIndex.Count) comparisons=$($comparisons.Count) max_regression_percent=10"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user