diff --git a/src/tools/bake_m2_cache.gd b/src/tools/bake_m2_cache.gd index e86ca59..adb68ad 100644 --- a/src/tools/bake_m2_cache.gd +++ b/src/tools/bake_m2_cache.gd @@ -176,14 +176,14 @@ func _bake_glb_animation_cache( if not force and FileAccess.file_exists(abs_out_glb): return true var abs_converter := ProjectSettings.globalize_path(converter) - var abs_output := ProjectSettings.globalize_path(output_dir) + var converter_output_directory := abs_out_glb.get_base_dir() if not FileAccess.file_exists(abs_converter): push_warning("M2 GLB converter not found: %s" % converter) return false var stdout := [] var exit_code := OS.execute( python_exe, - [abs_converter, abs_m2, abs_output], + [abs_converter, abs_m2, converter_output_directory], stdout, true, false) diff --git a/src/tools/verify_renderer_closeout_contracts.gd b/src/tools/verify_renderer_closeout_contracts.gd new file mode 100644 index 0000000..9ee9c26 --- /dev/null +++ b/src/tools/verify_renderer_closeout_contracts.gd @@ -0,0 +1,173 @@ +extends SceneTree + +## M03 closeout gate for cache versions and worker/main-thread render boundaries. + +const LOADER_PATH := "res://src/scenes/streaming/streaming_world_loader.gd" +const MANIFEST_PATH := "res://src/tools/render_baseline_manifest.json" +const M2_CACHE_BAKER_PATH := "res://src/tools/bake_m2_cache.gd" +const WORKER_FUNCTIONS: Array[String] = [ + "_load_tile_task", + "_load_terrain_splat_task", + "_load_tile_water_task", + "_group_tile_m2_task", +] +const WORKER_FORBIDDEN_TOKENS: Array[String] = [ + "RenderingServer", + ".add_child(", + ".queue_free(", + ".free(", + "Node3D.new(", + "MeshInstance3D.new(", + "MultiMeshInstance3D.new(", + "ArrayMesh.new(", + "build_tile_water_scene(", + "build_tile_coarse_render_payload(", +] +const MAIN_THREAD_FRAME_STEPS: Array[String] = [ + "_drain_tile_load_results()", + "_drain_terrain_upgrade_results()", + "_drain_terrain_control_splat_cache_results()", + "_drain_terrain_splat_cache_results()", + "_drain_terrain_splat_results()", + "_process_water_load_queue()", + "_drain_water_load_results()", + "_process_queues()", + "_drain_m2_group_results()", + "_drain_m2_animation_loads()", + "_drain_m2_mesh_loads()", + "_process_m2_build_jobs()", + "_process_wmo_build_jobs()", + "_process_wmo_render_build_jobs()", + "_process_detail_asset_queue()", +] + + +func _initialize() -> void: + var failures: Array[String] = [] + var loader_source := _read_text(LOADER_PATH, failures) + var process_source := _function_source(loader_source, "_process", failures) + _verify_frame_entrypoint(process_source, failures) + _verify_worker_boundaries(loader_source, failures) + _verify_rendering_server_boundary(loader_source, failures) + _verify_cache_contract(loader_source, failures) + _verify_m2_glb_cache_output_contract(failures) + + if not failures.is_empty(): + for failure in failures: + push_error("RENDERER_CLOSEOUT_CONTRACTS: %s" % failure) + quit(1) + return + + print("RENDERER_CLOSEOUT_CONTRACTS PASS workers=%d frame_steps=%d cache_versions=7" % [ + WORKER_FUNCTIONS.size(), + MAIN_THREAD_FRAME_STEPS.size(), + ]) + quit(0) + + +func _verify_frame_entrypoint(process_source: String, failures: Array[String]) -> void: + var scheduler_index := process_source.find("_render_budget_scheduler.begin_frame(") + if scheduler_index < 0: + failures.append("_process does not begin a render-budget frame") + return + for step in MAIN_THREAD_FRAME_STEPS: + var step_index := process_source.find(step) + if step_index < 0: + failures.append("_process is missing main-thread step %s" % step) + elif step_index < scheduler_index: + failures.append("%s runs before render-budget frame admission" % step) + + +func _verify_worker_boundaries(loader_source: String, failures: Array[String]) -> void: + for function_name in WORKER_FUNCTIONS: + var function_source := _function_source(loader_source, function_name, failures) + for forbidden_token in WORKER_FORBIDDEN_TOKENS: + if function_source.contains(forbidden_token): + failures.append("worker %s contains main-thread token %s" % [ + function_name, + forbidden_token, + ]) + + +func _verify_rendering_server_boundary(loader_source: String, failures: Array[String]) -> void: + var remaining_source := loader_source + for function_name in ["_create_render_instance", "_free_render_instance"]: + var function_source := _function_source(loader_source, function_name, failures) + remaining_source = remaining_source.replace(function_source, "") + if remaining_source.contains("RenderingServer.instance_create("): + failures.append("RenderingServer instance creation escaped the owned adapter") + if remaining_source.contains("RenderingServer.free_rid("): + failures.append("RenderingServer RID release escaped the owned adapter") + + +func _verify_cache_contract(loader_source: String, failures: Array[String]) -> void: + var manifest_source := _read_text(MANIFEST_PATH, failures) + var parsed_manifest = JSON.parse_string(manifest_source) + if not (parsed_manifest is Dictionary): + failures.append("baseline manifest is not a Dictionary") + return + var cache_contract: Dictionary = parsed_manifest.get("cache_contract", {}) + var expected_versions := { + "baked_terrain": 5, + "streaming_terrain": 2, + "terrain_splat": 1, + "terrain_control_splat": 3, + "wmo_streaming": 2, + "wmo_builder": 2, + "m2_material": 2, + } + if cache_contract.size() != expected_versions.size(): + failures.append("cache contract key count changed: expected=%d actual=%d" % [ + expected_versions.size(), + cache_contract.size(), + ]) + for cache_name in expected_versions: + if not cache_contract.has(cache_name): + failures.append("cache contract is missing %s" % cache_name) + continue + var cache_record: Dictionary = cache_contract[cache_name] + if int(cache_record.get("version", -1)) != int(expected_versions[cache_name]): + failures.append("cache version changed for %s" % cache_name) + + var required_loader_declarations := [ + "const REQUIRED_BAKED_TILE_FORMAT_VERSION := 5", + "const REQUIRED_SPLAT_TILE_FORMAT_VERSION := 1", + "const REQUIRED_CONTROL_SPLAT_TILE_FORMAT_VERSION := 3", + "STREAMING_TILE_SCRIPT.FORMAT_VERSION", + "WMO_STREAMING_SCRIPT.FORMAT_VERSION", + ] + for declaration in required_loader_declarations: + if not loader_source.contains(declaration): + failures.append("loader cache-version boundary is missing %s" % declaration) + + +func _verify_m2_glb_cache_output_contract(failures: Array[String]) -> void: + var baker_source := _read_text(M2_CACHE_BAKER_PATH, failures) + if not baker_source.contains( + "var converter_output_directory := abs_out_glb.get_base_dir()"): + failures.append("M2 GLB converter output is not derived from the nested cache path") + if not baker_source.contains( + "[abs_converter, abs_m2, converter_output_directory]"): + failures.append("M2 GLB converter does not receive the nested output directory") + + +func _function_source(source: String, function_name: String, failures: Array[String]) -> String: + var signature := "func %s(" % function_name + var start_index := source.find(signature) + if start_index < 0: + failures.append("missing function %s" % function_name) + return "" + var next_function_index := source.find("\nfunc ", start_index + signature.length()) + if next_function_index < 0: + return source.substr(start_index) + return source.substr(start_index, next_function_index - start_index) + + +func _read_text(path: String, failures: Array[String]) -> String: + var file := FileAccess.open(path, FileAccess.READ) + if file == null: + failures.append("cannot read %s" % path) + return "" + var content := file.get_as_text() + file.close() + return content diff --git a/src/tools/verify_renderer_closeout_contracts.gd.uid b/src/tools/verify_renderer_closeout_contracts.gd.uid new file mode 100644 index 0000000..c81191d --- /dev/null +++ b/src/tools/verify_renderer_closeout_contracts.gd.uid @@ -0,0 +1 @@ +uid://nqj0mu6r8omf diff --git a/tools/compare_render_performance.ps1 b/tools/compare_render_performance.ps1 new file mode 100644 index 0000000..36d3dd0 --- /dev/null +++ b/tools/compare_render_performance.ps1 @@ -0,0 +1,231 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$BaselineReport, + + [Parameter(Mandatory = $true)] + [string]$CandidateReport, + + [string]$OutputReport +) + +$ErrorActionPreference = 'Stop' +$failures = [System.Collections.Generic.List[string]]::new() +$comparisons = [System.Collections.Generic.List[object]]::new() + +function Read-RenderReport { + param( + [Parameter(Mandatory = $true)] + [string]$Path, + + [Parameter(Mandatory = $true)] + [string]$Label + ) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "$Label report does not exist: $Path" + } + $report = Get-Content -Raw -Encoding UTF8 -LiteralPath $Path | ConvertFrom-Json + if ($null -eq $report -or $null -eq $report.results) { + throw "$Label report has no results array: $Path" + } + return $report +} + +function Convert-ToStableJson { + param([object]$Value) + return $Value | ConvertTo-Json -Depth 20 -Compress +} + +function Assert-EqualValue { + param( + [string]$Name, + [object]$BaselineValue, + [object]$CandidateValue + ) + + $baselineJson = Convert-ToStableJson $BaselineValue + $candidateJson = Convert-ToStableJson $CandidateValue + if ($baselineJson -cne $candidateJson) { + $failures.Add("$Name differs: baseline=$baselineJson candidate=$candidateJson") + } +} + +function Get-ResultKey { + param([object]$Result) + return "$($Result.name)|$($Result.pass)" +} + +function Assert-CompatibleCacheInventory { + param( + [object]$BaselineInventory, + [object]$CandidateInventory + ) + + $baselineNames = @($BaselineInventory.PSObject.Properties.Name | Sort-Object) + $candidateNames = @($CandidateInventory.PSObject.Properties.Name | Sort-Object) + Assert-EqualValue 'cache inventory key set' $baselineNames $candidateNames + foreach ($cacheName in $baselineNames) { + $baselineRecord = $BaselineInventory.$cacheName + $candidateRecord = $CandidateInventory.$cacheName + if ($null -eq $candidateRecord) { + continue + } + Assert-EqualValue "cache inventory presence for $cacheName" ` + $baselineRecord.present ` + $candidateRecord.present + if ([bool]$baselineRecord.present -and [int64]$candidateRecord.file_count -le 0) { + $failures.Add("cache inventory is empty for $cacheName") + } + } +} + +function Build-ResultIndex { + param( + [object[]]$Results, + [string]$Label + ) + + $index = @{} + foreach ($result in $Results) { + $key = Get-ResultKey $result + if ($index.ContainsKey($key)) { + $failures.Add("$Label report contains duplicate result: $key") + continue + } + $index[$key] = $result + } + return $index +} + +function Compare-Metric { + param( + [string]$ResultKey, + [string]$MetricName, + [double]$BaselineValue, + [double]$CandidateValue, + [double]$AllowedRegressionPercent + ) + + $limit = if ($BaselineValue -eq 0.0) { + 0.0 + } else { + $BaselineValue * (1.0 + $AllowedRegressionPercent / 100.0) + } + $regressionPercent = if ($BaselineValue -eq 0.0) { + if ($CandidateValue -eq 0.0) { 0.0 } else { [double]::PositiveInfinity } + } else { + (($CandidateValue - $BaselineValue) / $BaselineValue) * 100.0 + } + $passed = $CandidateValue -le $limit + $comparisons.Add([pscustomobject]@{ + result = $ResultKey + metric = $MetricName + baseline = $BaselineValue + candidate = $CandidateValue + limit = $limit + regression_percent = $regressionPercent + passed = $passed + }) + if (-not $passed) { + $failures.Add( + ("{0} {1} regressed by {2:N3}%: baseline={3:N3} candidate={4:N3} limit={5:N3}" -f + $ResultKey, + $MetricName, + $regressionPercent, + $BaselineValue, + $CandidateValue, + $limit) + ) + } +} + +$baseline = Read-RenderReport -Path $BaselineReport -Label 'Baseline' +$candidate = Read-RenderReport -Path $CandidateReport -Label 'Candidate' + +Assert-EqualValue 'schema_version' $baseline.schema_version $candidate.schema_version +Assert-EqualValue 'profile' $baseline.profile $candidate.profile +Assert-EqualValue 'cache_state' $baseline.cache_state $candidate.cache_state +Assert-EqualValue 'viewport' $baseline.environment.viewport $candidate.environment.viewport +Assert-EqualValue 'Godot version' $baseline.environment.godot_version.string $candidate.environment.godot_version.string +Assert-EqualValue 'rendering driver' $baseline.environment.rendering_driver $candidate.environment.rendering_driver +Assert-EqualValue 'rendering method' $baseline.environment.rendering_method $candidate.environment.rendering_method +Assert-EqualValue 'video adapter' $baseline.environment.video_adapter $candidate.environment.video_adapter +Assert-EqualValue 'CPU' $baseline.environment.cpu $candidate.environment.cpu +Assert-EqualValue 'cache contract' $baseline.cache_contract $candidate.cache_contract +Assert-CompatibleCacheInventory $baseline.cache_inventory $candidate.cache_inventory + +$baselineIndex = Build-ResultIndex -Results @($baseline.results) -Label 'Baseline' +$candidateIndex = Build-ResultIndex -Results @($candidate.results) -Label 'Candidate' +Assert-EqualValue 'result key set' @($baselineIndex.Keys | Sort-Object) @($candidateIndex.Keys | Sort-Object) + +$budgets = $baseline.comparison_budgets +$metricDefinitions = @( + @('load_time_ms', 'load_time_ms_max_regression_percent', 'root'), + @('frame_ms_p95', 'frame_ms_p95_max_regression_percent', 'metrics'), + @('frame_ms_p99', 'frame_ms_p99_max_regression_percent', 'metrics'), + @('max_hitch_ms', 'max_hitch_ms_max_regression_percent', 'metrics'), + @('memory_static_bytes', 'memory_bytes_max_regression_percent', 'metrics'), + @('video_memory_bytes', 'memory_bytes_max_regression_percent', 'metrics') +) + +foreach ($key in @($baselineIndex.Keys | Sort-Object)) { + if (-not $candidateIndex.ContainsKey($key)) { + continue + } + $baselineResult = $baselineIndex[$key] + $candidateResult = $candidateIndex[$key] + foreach ($definition in $metricDefinitions) { + $metricName = $definition[0] + $budgetName = $definition[1] + $location = $definition[2] + $baselineValue = if ($location -eq 'root') { + [double]$baselineResult.$metricName + } else { + [double]$baselineResult.metrics.$metricName + } + $candidateValue = if ($location -eq 'root') { + [double]$candidateResult.$metricName + } else { + [double]$candidateResult.metrics.$metricName + } + Compare-Metric ` + -ResultKey $key ` + -MetricName $metricName ` + -BaselineValue $baselineValue ` + -CandidateValue $candidateValue ` + -AllowedRegressionPercent ([double]$budgets.$budgetName) + } +} + +$summary = [pscustomobject]@{ + schema_version = 1 + baseline_revision = $baseline.revision + candidate_revision = $candidate.revision + baseline_created_utc = $baseline.created_utc + candidate_created_utc = $candidate.created_utc + result_pairs = $baselineIndex.Count + metric_comparisons = $comparisons.Count + passed = $failures.Count -eq 0 + failures = @($failures) + comparisons = @($comparisons) +} + +if ($OutputReport) { + $parent = Split-Path -Parent $OutputReport + if ($parent -and -not (Test-Path -LiteralPath $parent)) { + New-Item -ItemType Directory -Path $parent | Out-Null + } + $summary | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -LiteralPath $OutputReport +} + +if ($failures.Count -gt 0) { + foreach ($failure in $failures) { + Write-Error "RENDER_PERFORMANCE: $failure" -ErrorAction Continue + } + Write-Host "RENDER_PERFORMANCE FAIL result_pairs=$($baselineIndex.Count) comparisons=$($comparisons.Count) failures=$($failures.Count)" + exit 1 +} + +Write-Host "RENDER_PERFORMANCE PASS result_pairs=$($baselineIndex.Count) comparisons=$($comparisons.Count) max_regression_percent=10" +exit 0