extends SceneTree ## Compares original-client JPG/PNG references with OpenWC checkpoint PNGs. ## Usage: godot --headless --path . --script res://src/tools/compare_render_checkpoints.gd -- ## --reference --candidate [--output ] ## [--region x,y,width,height] ## [--pixel-threshold 0.04] [--mean-threshold 0.015] [--changed-ratio-threshold 0.01] ## Use --self-test for a synthetic regression check that writes only under user://. const REPORT_SCHEMA_VERSION := 1 const DEFAULT_PIXEL_THRESHOLD := 0.04 const DEFAULT_MEAN_THRESHOLD := 0.015 const DEFAULT_CHANGED_RATIO_THRESHOLD := 0.01 func _initialize() -> void: var arguments := _parse_arguments(OS.get_cmdline_user_args()) if arguments.has("error"): push_error("RENDER_CHECKPOINT_DIFF: %s" % arguments.error) quit(2) return if arguments.get("self_test", false): quit(_run_self_test()) return var reference_directory := String(arguments.get("reference", "")) var candidate_directory := String(arguments.get("candidate", "")) if reference_directory.is_empty() or candidate_directory.is_empty(): push_error("RENDER_CHECKPOINT_DIFF: --reference and --candidate are required") quit(2) return var report := _compare_directories(reference_directory, candidate_directory, arguments) var output_path := String(arguments.get("output", "")) if not output_path.is_empty() and not _write_report(output_path, report): quit(2) return print("RENDER_CHECKPOINT_DIFF %s compared=%d failed=%d missing=%d" % [ "PASS" if report.passed else "FAIL", report.compared_count, report.failed_count, report.missing_count, ]) quit(0 if report.passed else 1) func _parse_arguments(raw_arguments: PackedStringArray) -> Dictionary: var parsed := { "pixel_threshold": DEFAULT_PIXEL_THRESHOLD, "mean_threshold": DEFAULT_MEAN_THRESHOLD, "changed_ratio_threshold": DEFAULT_CHANGED_RATIO_THRESHOLD, } var index := 0 while index < raw_arguments.size(): var argument := raw_arguments[index] if argument == "--self-test": parsed.self_test = true index += 1 continue if argument not in ["--reference", "--candidate", "--output", "--only", "--region", "--pixel-threshold", "--mean-threshold", "--changed-ratio-threshold"]: return {"error": "unknown argument: %s" % argument} if index + 1 >= raw_arguments.size(): return {"error": "missing value for %s" % argument} var key := argument.trim_prefix("--").replace("-", "_") var value := raw_arguments[index + 1] if key == "region": var region_parse_result := _parse_comparison_region(value) if region_parse_result.has("error"): return region_parse_result parsed[key] = region_parse_result.region elif key.ends_with("threshold"): if not value.is_valid_float(): return {"error": "%s requires a number" % argument} parsed[key] = float(value) else: parsed[key] = value index += 2 return parsed func _parse_comparison_region(value: String) -> Dictionary: var components := value.split(",", false) if components.size() != 4: return {"error": "--region requires x,y,width,height"} for component in components: if not component.strip_edges().is_valid_int(): return {"error": "--region components must be integers"} var region := Rect2i( int(components[0]), int(components[1]), int(components[2]), int(components[3]) ) if region.position.x < 0 or region.position.y < 0 or region.size.x <= 0 or region.size.y <= 0: return {"error": "--region requires non-negative x/y and positive width/height"} return {"region": region} func _compare_directories(reference_directory: String, candidate_directory: String, options: Dictionary) -> Dictionary: reference_directory = ProjectSettings.globalize_path(reference_directory) candidate_directory = ProjectSettings.globalize_path(candidate_directory) var reference_files := _reference_image_file_names(reference_directory) var only_filter := String(options.get("only", "")).to_lower() var results: Array[Dictionary] = [] var failed_count := 0 var missing_count := 0 for file_name in reference_files: if not only_filter.is_empty() and not file_name.get_basename().to_lower().contains(only_filter): continue var reference_path := reference_directory.path_join(file_name) var checkpoint_name := file_name.get_basename() var candidate_file_names := _candidate_file_names(candidate_directory, checkpoint_name) if candidate_file_names.is_empty(): results.append({"checkpoint": checkpoint_name, "reference": file_name, "status": "missing_candidate"}) missing_count += 1 continue for candidate_file_name in candidate_file_names: var candidate_path := candidate_directory.path_join(candidate_file_name) var comparison := _compare_images(reference_path, candidate_path, options) comparison.checkpoint = checkpoint_name comparison.reference = file_name comparison.candidate = candidate_file_name results.append(comparison) if comparison.status != "passed": failed_count += 1 var no_reference_images := reference_files.is_empty() return { "schema_version": REPORT_SCHEMA_VERSION, "passed": not no_reference_images and failed_count == 0 and missing_count == 0, "reference_directory": reference_directory, "candidate_directory": candidate_directory, "pixel_threshold": options.pixel_threshold, "mean_threshold": options.mean_threshold, "changed_ratio_threshold": options.changed_ratio_threshold, "comparison_region": _region_array(options.get("region", Rect2i())), "compared_count": results.size() - missing_count, "failed_count": failed_count, "missing_count": missing_count, "no_reference_images": no_reference_images, "results": results, } func _compare_images(reference_path: String, candidate_path: String, options: Dictionary) -> Dictionary: var reference_image := Image.load_from_file(reference_path) var candidate_image := Image.load_from_file(candidate_path) if reference_image == null or candidate_image == null or reference_image.is_empty() or candidate_image.is_empty(): return {"status": "load_error"} if reference_image.get_size() != candidate_image.get_size(): return { "status": "size_mismatch", "reference_size": [reference_image.get_width(), reference_image.get_height()], "candidate_size": [candidate_image.get_width(), candidate_image.get_height()], } var comparison_region: Rect2i = options.get("region", Rect2i(Vector2i.ZERO, reference_image.get_size())) if comparison_region == Rect2i(): comparison_region = Rect2i(Vector2i.ZERO, reference_image.get_size()) var image_region := Rect2i(Vector2i.ZERO, reference_image.get_size()) if not image_region.encloses(comparison_region): return { "status": "region_out_of_bounds", "region": _region_array(comparison_region), "image_size": [reference_image.get_width(), reference_image.get_height()], } var error_sum := 0.0 var changed_pixel_count := 0 var pixel_count := comparison_region.size.x * comparison_region.size.y for y in range(comparison_region.position.y, comparison_region.end.y): for x in range(comparison_region.position.x, comparison_region.end.x): var perceptual_error := _perceptual_color_error(reference_image.get_pixel(x, y), candidate_image.get_pixel(x, y)) error_sum += perceptual_error if perceptual_error > float(options.pixel_threshold): changed_pixel_count += 1 var mean_perceptual_error := error_sum / float(pixel_count) var changed_pixel_ratio := float(changed_pixel_count) / float(pixel_count) var passed := mean_perceptual_error <= float(options.mean_threshold) and changed_pixel_ratio <= float(options.changed_ratio_threshold) return { "status": "passed" if passed else "threshold_exceeded", "width": reference_image.get_width(), "height": reference_image.get_height(), "region": _region_array(comparison_region), "mean_perceptual_error": mean_perceptual_error, "changed_pixel_ratio": changed_pixel_ratio, } func _region_array(region: Rect2i) -> Array[int]: if region == Rect2i(): return [] return [region.position.x, region.position.y, region.size.x, region.size.y] func _perceptual_color_error(reference_color: Color, candidate_color: Color) -> float: var reference_linear := reference_color.srgb_to_linear() var candidate_linear := candidate_color.srgb_to_linear() var red_delta := absf(reference_linear.r - candidate_linear.r) var green_delta := absf(reference_linear.g - candidate_linear.g) var blue_delta := absf(reference_linear.b - candidate_linear.b) var alpha_delta := absf(reference_linear.a - candidate_linear.a) return 0.2126 * red_delta + 0.7152 * green_delta + 0.0722 * blue_delta + 0.25 * alpha_delta func _reference_image_file_names(directory_path: String) -> Array[String]: var directory := DirAccess.open(directory_path) if directory == null: return [] var file_names: Array[String] = [] for file_name in directory.get_files(): var lower_name := file_name.to_lower() if not lower_name.begins_with("diagnostic_") and (lower_name.ends_with(".png") or lower_name.ends_with(".jpg") or lower_name.ends_with(".jpeg")): file_names.append(file_name) file_names.sort() return file_names func _candidate_file_names(candidate_directory: String, checkpoint_name: String) -> Array[String]: var candidates: Array[String] = [] for pass_name in ["cold_process", "warm_revisit"]: var file_name := "%s__%s.png" % [checkpoint_name, pass_name] if FileAccess.file_exists(candidate_directory.path_join(file_name)): candidates.append(file_name) if candidates.is_empty(): var exact_file_name := "%s.png" % checkpoint_name if FileAccess.file_exists(candidate_directory.path_join(exact_file_name)): candidates.append(exact_file_name) return candidates func _write_report(output_path: String, report: Dictionary) -> bool: var absolute_path := ProjectSettings.globalize_path(output_path) var error := DirAccess.make_dir_recursive_absolute(absolute_path.get_base_dir()) if error != OK: push_error("RENDER_CHECKPOINT_DIFF: cannot create report directory: %s" % error_string(error)) return false var report_file := FileAccess.open(absolute_path, FileAccess.WRITE) if report_file == null: push_error("RENDER_CHECKPOINT_DIFF: cannot write report: %s" % output_path) return false report_file.store_string(JSON.stringify(report, " ")) return true func _run_self_test() -> int: var root := "user://render_checkpoint_diff_self_test" var reference_directory := root.path_join("reference") var identical_directory := root.path_join("identical") var changed_directory := root.path_join("changed") for directory_path in [reference_directory, identical_directory, changed_directory]: DirAccess.make_dir_recursive_absolute(ProjectSettings.globalize_path(directory_path)) _clear_directory_files(directory_path) var reference_image := Image.create(2, 2, false, Image.FORMAT_RGBA8) reference_image.fill(Color(0.25, 0.5, 0.75, 1.0)) var changed_image := reference_image.duplicate() changed_image.set_pixel(0, 0, Color.WHITE) if reference_image.save_jpg(reference_directory.path_join("synthetic.jpg"), 1.0) != OK: push_error("RENDER_CHECKPOINT_DIFF SELF_TEST: cannot save JPG reference") return 1 for pass_name in ["cold_process", "warm_revisit"]: if reference_image.save_png(identical_directory.path_join("synthetic__%s.png" % pass_name)) != OK: return 1 if changed_image.save_png(changed_directory.path_join("synthetic__%s.png" % pass_name)) != OK: return 1 var options := { "pixel_threshold": DEFAULT_PIXEL_THRESHOLD, "mean_threshold": DEFAULT_MEAN_THRESHOLD, "changed_ratio_threshold": DEFAULT_CHANGED_RATIO_THRESHOLD, } var identical_report := _compare_directories(reference_directory, identical_directory, options) var changed_report := _compare_directories(reference_directory, changed_directory, options) if not identical_report.passed or changed_report.passed: push_error("RENDER_CHECKPOINT_DIFF SELF_TEST: expected identical pass and changed failure") return 1 var region_options := options.duplicate() region_options["region"] = Rect2i(1, 1, 1, 1) var excluded_change_report := _compare_directories(reference_directory, changed_directory, region_options) if not excluded_change_report.passed or excluded_change_report.comparison_region != [1, 1, 1, 1]: push_error("RENDER_CHECKPOINT_DIFF SELF_TEST: region must exclude pixels outside the rectangle") return 1 var out_of_bounds_options := options.duplicate() out_of_bounds_options["region"] = Rect2i(1, 1, 2, 2) var out_of_bounds_result := _compare_images( reference_directory.path_join("synthetic.jpg"), changed_directory.path_join("synthetic__cold_process.png"), out_of_bounds_options ) if out_of_bounds_result.status != "region_out_of_bounds": push_error("RENDER_CHECKPOINT_DIFF SELF_TEST: out-of-bounds region must fail") return 1 if reference_image.save_jpg(reference_directory.path_join("ignored.jpg"), 1.0) != OK: return 1 var filtered_options := options.duplicate() filtered_options["only"] = "synthetic" var filtered_report := _compare_directories(reference_directory, identical_directory, filtered_options) if not filtered_report.passed or filtered_report.compared_count != 2: push_error("RENDER_CHECKPOINT_DIFF SELF_TEST: --only must exclude unrelated references") return 1 print("RENDER_CHECKPOINT_DIFF SELF_TEST PASS") return 0 func _clear_directory_files(directory_path: String) -> void: var directory := DirAccess.open(directory_path) if directory == null: return for file_name in directory.get_files(): directory.remove(file_name)