From a4f60dcb06c06592ded6f0eb04e6aeb5f2f88e98 Mon Sep 17 00:00:00 2001 From: sindoring Date: Sat, 11 Jul 2026 09:38:40 +0400 Subject: [PATCH] test(M00): add checkpoint perceptual comparison Work-Package: M00-QAR-VISUAL-DIFF-001 Agent: sindo-main-codex Tests: synthetic visual diff, M00 dry-run, coordination and documentation gates Fidelity: automates paired metrics; approved 3.3.5a captures and human approval remain required --- RENDER.md | 2 + .../claims/M00-QAR-VISUAL-DIFF-001.md | 78 +++++++ docs/RENDER_BASELINE.md | 40 +++- docs/modules/world-renderer.md | 7 +- src/tools/compare_render_checkpoints.gd | 209 ++++++++++++++++++ src/tools/render_baseline_manifest.json | 5 +- src/tools/verify_render_baseline_manifest.gd | 5 + tools/run_render_baseline.ps1 | 15 ++ 8 files changed, 356 insertions(+), 5 deletions(-) create mode 100644 coordination/claims/M00-QAR-VISUAL-DIFF-001.md create mode 100644 src/tools/compare_render_checkpoints.gd diff --git a/RENDER.md b/RENDER.md index 0a3dfa9..2423ad8 100644 --- a/RENDER.md +++ b/RENDER.md @@ -2,6 +2,8 @@ Воспроизводимый renderer baseline для текущей цели M00, regression manifest, метрики и правила сравнения описаны в [`docs/RENDER_BASELINE.md`](docs/RENDER_BASELINE.md). Baseline фиксирует текущее поведение и известные расхождения, но не заявляет parity с WoW 3.3.5a. +Парные checkpoint PNG можно автоматически проверить через `src/tools/compare_render_checkpoints.gd`; reference-снимки оригинального клиента остаются вне Git, а автоматический tolerance не заменяет human fidelity approval. + Цель renderer-работы в этом проекте: добиться ощущения производительности оригинального клиента WoW 3.3.5a в Godot, без видимых фризов при переходе ADT -> ADT и без постоянного отката видимых участков к низкому качеству. Этот документ фиксирует текущее состояние рендера, сделанные оптимизации и практические правила дальнейшей работы. diff --git a/coordination/claims/M00-QAR-VISUAL-DIFF-001.md b/coordination/claims/M00-QAR-VISUAL-DIFF-001.md new file mode 100644 index 0000000..105016d --- /dev/null +++ b/coordination/claims/M00-QAR-VISUAL-DIFF-001.md @@ -0,0 +1,78 @@ +# M00-QAR-VISUAL-DIFF-001 — Renderer checkpoint visual diff + + + +## Ownership + +- Target: M00 +- Program: QAR +- Owner/Agent ID: sindo-main-codex +- Branch: `work/sindo-main-codex/m00-checkpoint-diff` +- Lease expires UTC: 2026-07-13 +- Integrator: milestone integrator + +## Outcome + +Provide a deterministic, headless perceptual comparison for paired renderer checkpoint PNGs, with machine-readable results and synthetic regression coverage. + +## Non-goals + +- Supplying or committing original-client screenshots. +- Claiming visual parity or replacing human approval. +- Image registration, camera correction, or temporal animation matching. + +## Paths + +- Exclusive: `src/tools/compare_render_checkpoints.gd` +- Shared/hotspots: `src/tools/render_baseline_manifest.json`, `tools/run_render_baseline.ps1`, `docs/RENDER_BASELINE.md`, `docs/modules/world-renderer.md`, `RENDER.md` +- Generated/ignored: comparison reports, diff PNGs, original-client captures + +## Contracts and data + +- Public API/events: headless CLI arguments and JSON report documented in renderer baseline docs +- Schema/format version: comparison report schema 1; baseline manifest schema remains 1 +- Migration/compatibility: additive manifest budget fields +- Consumers: developers and CI + +## Dependencies + +- Requires: Godot `Image` API and M00 checkpoint naming contract +- Blocks: automated portion of original-client paired comparison +- External state: approved build 12340 screenshots remain unavailable + +## Verification + +- Commands: headless self-test; manifest verification; documentation and coordination gates +- Fixtures: tool-generated identical and changed 2x2 PNGs +- Fidelity evidence: algorithm is deterministic; real-client evidence remains a manual/external input +- Performance budget: offline checkpoint operation, linear in pixel count + +## Documentation deliverables + +- Inline public API docs: CLI usage in script header +- Module specification: update renderer verification and source map +- Data-flow diagram: update baseline documentation +- Sequence/state/dependency diagrams: not stateful or asynchronous; not applicable +- Source map/status updates: renderer module and RENDER notes + +## Simplicity and naming + +- Important names introduced: `mean_perceptual_error`, `changed_pixel_ratio` +- Simplest considered solution: direct Godot Image iteration with no dependency +- Rejected complexity/abstractions: external image library, SSIM window pipeline, image registration +- Unavoidable complexity and justification: sRGB linearization avoids comparing encoded channel values directly +- Measured optimization evidence: not required for offline seven-checkpoint comparison + +## Status + +- State: ready +- Done: comparator, runner integration, manifest tolerance contract, synthetic regression, module/baseline documentation, repository gates +- Next: integrator review and paired run when approved build 12340 screenshots become available +- Blocked by: real paired screenshots only for final human fidelity approval + +## Handoff + +- Commit: branch HEAD +- Results: synthetic identical image passed, changed image failed; full M00 dry-run and repository gates passed +- Remaining risks: thresholds need calibration against approved paired captures; animation phase and camera alignment still require human review +- Documentation updated: `docs/RENDER_BASELINE.md`, `docs/modules/world-renderer.md`, `RENDER.md` diff --git a/docs/RENDER_BASELINE.md b/docs/RENDER_BASELINE.md index 53318f9..4f12ed9 100644 --- a/docs/RENDER_BASELINE.md +++ b/docs/RENDER_BASELINE.md @@ -24,6 +24,44 @@ - `__cold_process.png` — первый визит в текущем процессе; - `__warm_revisit.png` — повторный визит в том же процессе после обхода контрольных точек. +## Парное визуальное сравнение + +Одобренные снимки оригинального клиента хранятся вне Git. Если каталог с ними доступен локально, единый запуск может сразу сравнить PNG с одинаковыми именами: + +```powershell +.\tools\run_render_baseline.ps1 ` + -ReferenceCheckpointDirectory 'D:\private-fixtures\wow-3.3.5a-checkpoints' +``` + +Результат записывается в `user://render_baseline/visual_comparison.json`. Другой путь задаётся через `-VisualComparisonReport`. `-DryRun` несовместим с визуальным сравнением, потому что не создаёт PNG. + +Отдельный запуск сравнения: + +```powershell +godot --headless --path . ` + --script res://src/tools/compare_render_checkpoints.gd -- ` + --reference 'D:\private-fixtures\wow-3.3.5a-checkpoints' ` + --candidate "$env:APPDATA\Godot\app_userdata\OpenWC\render_baseline" ` + --output user://render_baseline/visual_comparison.json +``` + +Сравнение переводит sRGB в linear color space, вычисляет взвешенную ошибку яркости/цвета для каждого пикселя, среднюю ошибку кадра и долю пикселей выше локального порога. Нормативные defaults находятся в `comparison_budgets` manifest: mean error `0.015`, changed-pixel ratio `0.01`, pixel error threshold `0.04`. Любое превышение, несовпадение размеров, отсутствие пары или пустой reference-каталог возвращает ненулевой exit code. + +Автоматический pass не является доказательством parity: animation phase, weather, camera alignment и intentional gaps требуют human approval. Для проверки алгоритма без proprietary данных используется: + +```powershell +godot --headless --path . --script res://src/tools/compare_render_checkpoints.gd -- --self-test +``` + +```mermaid +flowchart LR + R[Private approved reference PNGs] --> C[Checkpoint comparator] + N[New baseline PNGs] --> C + M[Manifest tolerance defaults] --> C + C --> J[JSON metrics and pass/fail] + J --> H[Human fidelity approval] +``` + `cold_process` не означает очищенный Windows filesystem cache. Поле `cache_state` и полный inventory обязаны интерпретироваться вместе с результатом. Удаление или принудительная пересборка cache не входит в baseline-команду. Для отдельного запуска после контролируемой очистки безопасного локального cache следует передать осмысленную метку, например `-CacheState rebuilt-clean`; proprietary source assets команда не изменяет. ## Контрольные точки и детерминизм @@ -38,7 +76,7 @@ - синтетический native-animation probe `GryphonRoost01.m2`; - тот же Elwynn overview в 19:00 для sky transition. -Зафиксированы camera/target/player position, viewport, FOV, quality preset и время мира. Для temporal shader/animation кадров SHA-256 служит идентификатором артефакта, но не pass-критерием: будущая автоматическая визуальная проверка должна использовать perceptual diff и согласованный tolerance. Синтетический animation probe существует только внутри capture tool и не меняет runtime streaming scene. +Зафиксированы camera/target/player position, viewport, FOV, quality preset и время мира. Для temporal shader/animation кадров SHA-256 служит идентификатором артефакта, но не самостоятельным pass-критерием: автоматическая визуальная проверка использует perceptual diff и согласованный tolerance. Синтетический animation probe существует только внутри capture tool и не меняет runtime streaming scene. ## Метрики и budget diff --git a/docs/modules/world-renderer.md b/docs/modules/world-renderer.md index 3f07527..f1f2ff0 100644 --- a/docs/modules/world-renderer.md +++ b/docs/modules/world-renderer.md @@ -7,7 +7,7 @@ | Status | Partial | | Target/work package | M00 active; декомпозиция M01–M03 | | Owners | Renderer workstream / milestone integrator | -| Last verified | `93bfe11` + M00 worktree baseline, 2026-07-10 | +| Last verified | M00 visual-diff worktree, 2026-07-11 | | Profiles/capabilities | `Performance`, `Balanced`, `High`, `Custom`; Blizzlike fidelity incomplete | ## Purpose @@ -192,9 +192,9 @@ Exact exported settings and cache versions remain documented in [`../../RENDER.m ## Verification -- Unit/contract tests: material mapping, unique-ID dedupe, placement probes, baseline manifest. +- Unit/contract tests: material mapping, unique-ID dedupe, placement probes, baseline manifest, synthetic perceptual checkpoint diff. - Integration/E2E: Eastern Kingdoms/Kalimdor streaming scenes and seven cold/warm checkpoints. -- Fidelity evidence: current fixed-position captures exist; approved original-client paired screenshots are still missing. +- Fidelity evidence: current fixed-position captures and automated paired-PNG metrics exist; approved original-client paired screenshots and human approval are still missing. - Performance budgets: M00 report records cold/warm p95 and max hitch; no final acceptance threshold yet. - Manual diagnostics: [`../RENDER_BASELINE.md`](../RENDER_BASELINE.md) and [`../../RENDER.md`](../../RENDER.md). @@ -239,6 +239,7 @@ Exact exported settings and cache versions remain documented in [`../../RENDER.m | `src/native/src/*_loader.cpp` | Native binary parsing | | `src/tools/build_*cache.gd`, `src/tools/bake_*cache.gd` | Offline cache generation | | `tools/run_render_baseline.ps1` | Unified M00 baseline runner | +| `src/tools/compare_render_checkpoints.gd` | Offline paired-PNG perceptual metrics and JSON pass/fail report | ## Related decisions and references diff --git a/src/tools/compare_render_checkpoints.gd b/src/tools/compare_render_checkpoints.gd new file mode 100644 index 0000000..d8da8e2 --- /dev/null +++ b/src/tools/compare_render_checkpoints.gd @@ -0,0 +1,209 @@ +extends SceneTree + +## Compares paired PNG checkpoint directories without storing proprietary reference images. +## Usage: godot --headless --path . --script res://src/tools/compare_render_checkpoints.gd -- +## --reference --candidate [--output ] +## [--pixel-threshold 0.04] [--mean-threshold 0.015] [--changed-ratio-threshold 0.01] +## Use --self-test for a synthetic regression check that writes only under user://. + +const REPORT_SCHEMA_VERSION := 1 +const DEFAULT_PIXEL_THRESHOLD := 0.04 +const DEFAULT_MEAN_THRESHOLD := 0.015 +const DEFAULT_CHANGED_RATIO_THRESHOLD := 0.01 + + +func _initialize() -> void: + var arguments := _parse_arguments(OS.get_cmdline_user_args()) + if arguments.has("error"): + push_error("RENDER_CHECKPOINT_DIFF: %s" % arguments.error) + quit(2) + return + if arguments.get("self_test", false): + quit(_run_self_test()) + return + + var reference_directory := String(arguments.get("reference", "")) + var candidate_directory := String(arguments.get("candidate", "")) + if reference_directory.is_empty() or candidate_directory.is_empty(): + push_error("RENDER_CHECKPOINT_DIFF: --reference and --candidate are required") + quit(2) + return + + var report := _compare_directories(reference_directory, candidate_directory, arguments) + var output_path := String(arguments.get("output", "")) + if not output_path.is_empty() and not _write_report(output_path, report): + quit(2) + return + print("RENDER_CHECKPOINT_DIFF %s compared=%d failed=%d missing=%d" % [ + "PASS" if report.passed else "FAIL", + report.compared_count, + report.failed_count, + report.missing_count, + ]) + quit(0 if report.passed else 1) + + +func _parse_arguments(raw_arguments: PackedStringArray) -> Dictionary: + var parsed := { + "pixel_threshold": DEFAULT_PIXEL_THRESHOLD, + "mean_threshold": DEFAULT_MEAN_THRESHOLD, + "changed_ratio_threshold": DEFAULT_CHANGED_RATIO_THRESHOLD, + } + var index := 0 + while index < raw_arguments.size(): + var argument := raw_arguments[index] + if argument == "--self-test": + parsed.self_test = true + index += 1 + continue + if argument not in ["--reference", "--candidate", "--output", "--pixel-threshold", "--mean-threshold", "--changed-ratio-threshold"]: + return {"error": "unknown argument: %s" % argument} + if index + 1 >= raw_arguments.size(): + return {"error": "missing value for %s" % argument} + var key := argument.trim_prefix("--").replace("-", "_") + var value := raw_arguments[index + 1] + if key.ends_with("threshold"): + if not value.is_valid_float(): + return {"error": "%s requires a number" % argument} + parsed[key] = float(value) + else: + parsed[key] = value + index += 2 + return parsed + + +func _compare_directories(reference_directory: String, candidate_directory: String, options: Dictionary) -> Dictionary: + var reference_files := _png_file_names(reference_directory) + var results: Array[Dictionary] = [] + var failed_count := 0 + var missing_count := 0 + for file_name in reference_files: + var reference_path := reference_directory.path_join(file_name) + var candidate_path := candidate_directory.path_join(file_name) + if not FileAccess.file_exists(candidate_path): + results.append({"name": file_name, "status": "missing_candidate"}) + missing_count += 1 + continue + var comparison := _compare_images(reference_path, candidate_path, options) + comparison.name = file_name + results.append(comparison) + if comparison.status != "passed": + failed_count += 1 + var no_reference_images := reference_files.is_empty() + return { + "schema_version": REPORT_SCHEMA_VERSION, + "passed": not no_reference_images and failed_count == 0 and missing_count == 0, + "reference_directory": reference_directory, + "candidate_directory": candidate_directory, + "pixel_threshold": options.pixel_threshold, + "mean_threshold": options.mean_threshold, + "changed_ratio_threshold": options.changed_ratio_threshold, + "compared_count": results.size() - missing_count, + "failed_count": failed_count, + "missing_count": missing_count, + "no_reference_images": no_reference_images, + "results": results, + } + + +func _compare_images(reference_path: String, candidate_path: String, options: Dictionary) -> Dictionary: + var reference_image := Image.load_from_file(reference_path) + var candidate_image := Image.load_from_file(candidate_path) + if reference_image == null or candidate_image == null or reference_image.is_empty() or candidate_image.is_empty(): + return {"status": "load_error"} + if reference_image.get_size() != candidate_image.get_size(): + return { + "status": "size_mismatch", + "reference_size": [reference_image.get_width(), reference_image.get_height()], + "candidate_size": [candidate_image.get_width(), candidate_image.get_height()], + } + + var error_sum := 0.0 + var changed_pixel_count := 0 + var pixel_count := reference_image.get_width() * reference_image.get_height() + for y in reference_image.get_height(): + for x in reference_image.get_width(): + var perceptual_error := _perceptual_color_error(reference_image.get_pixel(x, y), candidate_image.get_pixel(x, y)) + error_sum += perceptual_error + if perceptual_error > float(options.pixel_threshold): + changed_pixel_count += 1 + var mean_perceptual_error := error_sum / float(pixel_count) + var changed_pixel_ratio := float(changed_pixel_count) / float(pixel_count) + var passed := mean_perceptual_error <= float(options.mean_threshold) and changed_pixel_ratio <= float(options.changed_ratio_threshold) + return { + "status": "passed" if passed else "threshold_exceeded", + "width": reference_image.get_width(), + "height": reference_image.get_height(), + "mean_perceptual_error": mean_perceptual_error, + "changed_pixel_ratio": changed_pixel_ratio, + } + + +func _perceptual_color_error(reference_color: Color, candidate_color: Color) -> float: + var reference_linear := reference_color.srgb_to_linear() + var candidate_linear := candidate_color.srgb_to_linear() + var red_delta := absf(reference_linear.r - candidate_linear.r) + var green_delta := absf(reference_linear.g - candidate_linear.g) + var blue_delta := absf(reference_linear.b - candidate_linear.b) + var alpha_delta := absf(reference_linear.a - candidate_linear.a) + return 0.2126 * red_delta + 0.7152 * green_delta + 0.0722 * blue_delta + 0.25 * alpha_delta + + +func _png_file_names(directory_path: String) -> Array[String]: + var directory := DirAccess.open(directory_path) + if directory == null: + return [] + var file_names: Array[String] = [] + for file_name in directory.get_files(): + if file_name.to_lower().ends_with(".png"): + file_names.append(file_name) + file_names.sort() + return file_names + + +func _write_report(output_path: String, report: Dictionary) -> bool: + var absolute_path := ProjectSettings.globalize_path(output_path) + var error := DirAccess.make_dir_recursive_absolute(absolute_path.get_base_dir()) + if error != OK: + push_error("RENDER_CHECKPOINT_DIFF: cannot create report directory: %s" % error_string(error)) + return false + var report_file := FileAccess.open(absolute_path, FileAccess.WRITE) + if report_file == null: + push_error("RENDER_CHECKPOINT_DIFF: cannot write report: %s" % output_path) + return false + report_file.store_string(JSON.stringify(report, " ")) + return true + + +func _run_self_test() -> int: + var root := "user://render_checkpoint_diff_self_test" + var reference_directory := root.path_join("reference") + var identical_directory := root.path_join("identical") + var changed_directory := root.path_join("changed") + for directory_path in [reference_directory, identical_directory, changed_directory]: + DirAccess.make_dir_recursive_absolute(ProjectSettings.globalize_path(directory_path)) + var reference_image := Image.create(2, 2, false, Image.FORMAT_RGBA8) + reference_image.fill(Color(0.25, 0.5, 0.75, 1.0)) + var changed_image := reference_image.duplicate() + changed_image.set_pixel(0, 0, Color.WHITE) + for pair in [ + [reference_directory, reference_image], + [identical_directory, reference_image], + [changed_directory, changed_image], + ]: + var save_error: Error = pair[1].save_png(String(pair[0]).path_join("synthetic.png")) + if save_error != OK: + push_error("RENDER_CHECKPOINT_DIFF SELF_TEST: cannot save fixture") + return 1 + var options := { + "pixel_threshold": DEFAULT_PIXEL_THRESHOLD, + "mean_threshold": DEFAULT_MEAN_THRESHOLD, + "changed_ratio_threshold": DEFAULT_CHANGED_RATIO_THRESHOLD, + } + var identical_report := _compare_directories(reference_directory, identical_directory, options) + var changed_report := _compare_directories(reference_directory, changed_directory, options) + if not identical_report.passed or changed_report.passed: + push_error("RENDER_CHECKPOINT_DIFF SELF_TEST: expected identical pass and changed failure") + return 1 + print("RENDER_CHECKPOINT_DIFF SELF_TEST PASS") + return 0 diff --git a/src/tools/render_baseline_manifest.json b/src/tools/render_baseline_manifest.json index 6a30d7c..8b63dec 100644 --- a/src/tools/render_baseline_manifest.json +++ b/src/tools/render_baseline_manifest.json @@ -14,7 +14,10 @@ "max_hitch_ms_max_regression_percent": 10.0, "load_time_ms_max_regression_percent": 10.0, "memory_bytes_max_regression_percent": 10.0, - "visual_diff": "perceptual comparison required; exact PNG hashes are evidence, not an animated-scene pass criterion" + "visual_mean_perceptual_error_max": 0.015, + "visual_changed_pixel_ratio_max": 0.01, + "visual_pixel_error_threshold": 0.04, + "visual_diff": "automated thresholds require human approval; exact PNG hashes are evidence, not an animated-scene pass criterion" }, "required_coverage": [ "terrain", diff --git a/src/tools/verify_render_baseline_manifest.gd b/src/tools/verify_render_baseline_manifest.gd index 0ae1170..f206475 100644 --- a/src/tools/verify_render_baseline_manifest.gd +++ b/src/tools/verify_render_baseline_manifest.gd @@ -42,6 +42,11 @@ func _initialize() -> void: var required := String(required_variant) _expect(covered.has(required), "missing checkpoint coverage: %s" % required, failures) + var comparison_budgets: Dictionary = manifest.get("comparison_budgets", {}) + for threshold_name in ["visual_mean_perceptual_error_max", "visual_changed_pixel_ratio_max", "visual_pixel_error_threshold"]: + var threshold := float(comparison_budgets.get(threshold_name, -1.0)) + _expect(threshold >= 0.0 and threshold <= 1.0, "%s must be between 0 and 1" % threshold_name, failures) + var contract: Dictionary = manifest.get("cache_contract", {}) var actual_versions := { "baked_terrain": STREAMING_LOADER.REQUIRED_BAKED_TILE_FORMAT_VERSION, diff --git a/tools/run_render_baseline.ps1 b/tools/run_render_baseline.ps1 index 5a109ca..13ced1c 100644 --- a/tools/run_render_baseline.ps1 +++ b/tools/run_render_baseline.ps1 @@ -5,6 +5,8 @@ param( [string]$CacheState = "existing", [double]$WaitSeconds = 8.0, [double]$MeasureSeconds = 3.0, + [string]$ReferenceCheckpointDirectory, + [string]$VisualComparisonReport = "user://render_baseline/visual_comparison.json", [switch]$DryRun ) @@ -57,6 +59,19 @@ try { $captureArgs = @("--headless") + $captureArgs + @("--dry-run") } Invoke-GodotStep "capture" $captureArgs + if ($ReferenceCheckpointDirectory) { + if ($DryRun) { + throw "Reference checkpoint comparison requires PNG capture; remove -DryRun" + } + Invoke-GodotStep "visual-comparison" @( + "--headless", "--path", ".", + "--script", "res://src/tools/compare_render_checkpoints.gd", + "--", + "--reference", $ReferenceCheckpointDirectory, + "--candidate", $Output, + "--output", $VisualComparisonReport + ) + } Write-Host "Renderer baseline completed. Report: $Output/report.json" } finally { Pop-Location