diff --git a/RENDER.md b/RENDER.md index bc48cd3..9388981 100644 --- a/RENDER.md +++ b/RENDER.md @@ -30,6 +30,7 @@ Paired run 2026-07-11 подтвердил крупный coordinate/placement g - `src/render/wmo/wmo_placement_resolver.gd` - value-only WMO path, identity and transform rules. - `src/render/wmo/wmo_placement_registry.gd` - WMO placement-key to tile/global reference sets; Nodes and render jobs remain in the streamer. - `src/render/wmo/wmo_render_build_step_planner.gd` - mesh-first lightweight WMO group operation and cursor planning without Nodes or Resources. +- `src/render/wmo/wmo_render_build_queue.gd` / `wmo_render_build_job.gd` - typed pending group jobs, FIFO placement keys and strong root/resource references without engine destruction. - `src/scenes/streaming/eastern_kingdoms_streaming.tscn` - текущая сцена для проверки Azeroth/Eastern Kingdoms. - `addons/mpq_extractor/loaders/adt_builder.gd` - сборка ADT terrain, control splat материалов и MH2O liquids. - `addons/mpq_extractor/loaders/m2_builder.gd` - сборка статического M2 mesh/material. @@ -401,6 +402,8 @@ Known limits: the parallel key-to-Node map and performs final-release cancellation/free; - `WmoRenderBuildStepPlanner` selects one mesh or MultiMesh group and its next cursors; the streamer still materializes Nodes and consumes the frame permit; +- `WmoRenderBuildQueue` owns pending typed jobs/FIFO keys and releases references + on cancel/reset; the streamer still validates and frees engine objects; - большие WMO не должны инстанцироваться как тяжелые `.tscn` во время движения; - добавлен lightweight render-cache path для WMO groups; - `wmo_render_group_ops_per_tick` ограничивает подключение WMO groups по кадрам; diff --git a/docs/modules/README.md b/docs/modules/README.md index b56a0a2..6c57989 100644 --- a/docs/modules/README.md +++ b/docs/modules/README.md @@ -22,6 +22,7 @@ | WMO placement resolver | Implemented extraction | [`wmo-placement-resolver.md`](wmo-placement-resolver.md) | | WMO placement registry | Implemented extraction | [`wmo-placement-registry.md`](wmo-placement-registry.md) | | WMO render build step planner | Implemented extraction | [`wmo-render-build-step-planner.md`](wmo-render-build-step-planner.md) | +| WMO render build queue | Implemented extraction | [`wmo-render-build-queue.md`](wmo-render-build-queue.md) | | Third-person camera | Implemented | [`third-person-camera.md`](third-person-camera.md) | | Character presentation | Implemented boundary / Partial fidelity | [`character-presentation.md`](character-presentation.md) | | Renderer | Partial | [`world-renderer.md`](world-renderer.md), [`../../RENDER.md`](../../RENDER.md) | diff --git a/docs/modules/wmo-render-build-queue.md b/docs/modules/wmo-render-build-queue.md new file mode 100644 index 0000000..5b8a8ad --- /dev/null +++ b/docs/modules/wmo-render-build-queue.md @@ -0,0 +1,238 @@ +# WMO Render Build Queue + +## Metadata + +| Field | Value | +|---|---| +| Status | Implemented | +| Target/work package | M03 / `M03-RND-WMO-RENDER-BUILD-QUEUE-001` | +| Owners | Typed lightweight-WMO build jobs and FIFO placement-key queue | +| Last verified | Worktree `work/sindo-main-codex/m03-wmo-render-build-queue`, 2026-07-17 | +| Profiles/capabilities | Lightweight WMO render-cache path; profile-independent | + +## Purpose + +Own pending lightweight-WMO render build records and their FIFO placement-key +order outside the monolithic loader. Each typed job retains its placement key, +scene root, render resource and mesh/MultiMesh cursors while the loader performs +all validation, planning, materialization, permit consumption and destruction. + +## Non-goals + +- Free or queue-free Nodes, Resources or RIDs. +- Create MeshInstance3D/MultiMeshInstance3D or refresh materials. +- Own frame permits, placement references, caches or load requests. +- Change duplicate enqueue, stale-key cleanup, cancellation or ordering. +- Change cache versions, WMO transforms, shadows, visibility or fidelity rules. + +## Context and boundaries + +```mermaid +flowchart LR + Placement[Successful WMO placement] --> Loader[StreamingWorldLoader] + Root[Node3D root] --> Loader + Cache[WMOStreamingResource] --> Loader + Loader --> Queue[WmoRenderBuildQueue] + Queue --> Job[WmoRenderBuildJob] + Job --> Loader + Loader --> Planner[WmoRenderBuildStepPlanner] + Planner --> Step[Operation and next cursors] + Step --> Loader + Loader --> Scene[Main-thread group Node attach] + Loader --> Budget[RenderBudgetScheduler permit] +``` + +The queue may strongly reference Node3D and Resource through typed jobs, but it +must not destroy, mutate or inspect their render content. RenderingServer, RID, +Mesh/MultiMesh instances, files, worker threads, mutexes, gameplay, network and +editor UI are forbidden dependencies. + +## Public API + +### `WmoRenderBuildJob` + +| Symbol | Kind | Purpose | Thread/lifetime | Errors | +|---|---|---|---|---| +| constructor `(placement_key, root, render_resource)` | Constructor | Create cursor-zero job with strong references | Renderer main thread; until cancel/clear/replacement | Queue validates inputs before construction | +| `placement_key()` | Query | Return immutable queue identity | Any serialized caller; job lifetime | None | +| `root()` | Query | Borrow retained Node3D reference | Main thread; no ownership transfer | May later be invalid; loader checks | +| `render_resource()` | Query | Borrow retained Resource reference | Main thread; no ownership transfer | Queue does not validate cache contents | +| `mesh_index()` / `multimesh_index()` | Query | Return current group cursors | Serialized job drain | None | +| `adopt_cursors(next_mesh_index, next_multimesh_index)` | Command | Atomically adopt planner cursors | Renderer main thread | Raw integers are accepted | +| `diagnostic_snapshot()` | Query | Return detached scalar state and presence flags | Caller-owned result | Never exposes engine objects | + +### `WmoRenderBuildQueue` + +| Symbol | Kind | Purpose | Thread/lifetime | Errors | +|---|---|---|---|---| +| `enqueue(placement_key, root, render_resource)` | Command | Replace keyed job and append FIFO key | Renderer main thread; map session | Empty/null input returns false | +| `has_pending()` / `pending_count()` | Query | Observe FIFO entries including duplicate/stale keys | Renderer main thread | None | +| `active_job_count()` | Query | Count current keyed jobs | Renderer main thread | None | +| `front_key()` / `pop_front()` | Queue operation | Inspect/remove one front FIFO key | Renderer main thread | Empty queue returns empty String | +| `job_for(placement_key)` | Query | Borrow current typed job or null for stale key | Renderer main thread | No ownership transfer | +| `cancel(placement_key)` | Command/query | Remove keyed job and first matching FIFO key | Renderer main thread | Unknown key returns false; stale duplicates remain | +| `clear()` | Command | Release all job/key references idempotently | Reset/shutdown | Does not free engine objects | +| `diagnostic_snapshot()` | Query | Return detached FIFO order and key-sorted scalar jobs | Caller-owned result | None | + +## Inputs and outputs + +| Direction | Contract/data | Producer | Consumer | Ownership | Thread/lifetime | +|---|---|---|---|---|---| +| Input | Resolved placement key | Placement/build adapter | Queue/job | Copied String | Until cancel/clear/replacement | +| Input | WMO root Node3D | Loader instance adapter | Job | Strong reference; loader owns engine lifetime | Until job release | +| Input | Validated WMOStreamingResource | Loader cache adapter | Job | Strong Resource reference; cache/loader owns semantics | Until job release | +| Input | Next mesh/MultiMesh cursors | Build step planner through loader | Job | Integer values | Until next adoption | +| Output | FIFO front key and typed job | Queue | Loader drain | Borrowed references | One drain iteration | +| Output | Detached scalar diagnostics | Job/queue | Verifier/future facade adapter | Fresh values | Caller lifetime | + +Side effects are limited to reference retention/release, cursor mutation and +Dictionary/Array ordering. Scene tree, material, RID, file and permit effects +remain outside the module. + +## Data flow + +```mermaid +flowchart TD + Enqueue[enqueue key/root/resource] --> Valid{Inputs valid?} + Valid -->|no| Reject[false; unchanged] + Valid -->|yes| Replace[Replace keyed typed job] + Replace --> Append[Append key even when duplicate] + Drain[front key] --> Lookup{Current keyed job exists?} + Lookup -->|no stale| Pop[pop front and continue] + Lookup -->|yes| Borrow[Borrow job to loader] + Borrow --> Plan[Loader invokes step planner] + Plan --> Adopt[Job adopts both next cursors] + Cancel[cancel key] --> EraseJob[Erase keyed job] + EraseJob --> EraseFirst[Erase first matching FIFO key] +``` + +## Lifecycle/state + +The queue can be `Empty`, `Pending` or `StaleFront`. Duplicate enqueue followed +by cancel can intentionally produce `StaleFront`, matching the previous separate +Dictionary/Array behavior. The loader pops a stale front without consuming a +permit. `clear` returns every state to `Empty` without freeing engine objects. + +```mermaid +stateDiagram-v2 + [*] --> Empty + Empty --> Pending: valid enqueue + Pending --> Pending: enqueue / cursor adoption + Pending --> Empty: final cancel or clear + Pending --> StaleFront: duplicate key then cancel first occurrence + StaleFront --> Pending: pop stale, more live entries remain + StaleFront --> Empty: pop stale or clear +``` + +## Main sequence + +```mermaid +sequenceDiagram + participant Loader as StreamingWorldLoader + participant Queue as WmoRenderBuildQueue + participant Job as WmoRenderBuildJob + participant Planner as WmoRenderBuildStepPlanner + Loader->>Queue: enqueue(key, root, resource) + loop while permit and pending key + Loader->>Queue: front_key + job_for + alt stale key + Loader->>Queue: pop_front + else live job + Queue-->>Loader: borrowed Job + Loader->>Planner: plan_step(counts, job cursors) + Planner-->>Loader: operation and next cursors + Loader->>Loader: materialize one group on main thread + Loader->>Job: adopt_cursors(next mesh, next MultiMesh) + Loader->>Loader: consume one permit + end + end + Loader->>Queue: cancel(key) or clear() + Note over Queue: references released; Nodes/Resources not freed +``` + +## Ownership, threading and resources + +- Queue owns its job Dictionary and FIFO key Array. +- Job owns strong references, but loader owns Node/Resource semantic lifetime. +- Loader checks `is_instance_valid`, creates/frees Nodes and controls caches. +- Loader invokes planner and adopts cursor output on the main thread. +- Queue/job contain no mutex; all mutation is serialized by the renderer drain. +- `clear` may release the last Resource reference but never calls engine free APIs. + +## Errors, cancellation and recovery + +| Failure/state | Detection | Behavior | Diagnostic | Recovery | +|---|---|---|---|---| +| Empty key/null root/resource | Enqueue guards | Reject without mutation | Contract verifier | Repair caller input | +| Duplicate key | Existing keyed job | Replace job and append key | Queue diagnostics | Historical stale handling remains | +| Stale FIFO key | `job_for` returns null | Loader pops front without permit | Contract/source verifier | Continue drain | +| Root invalidated externally | Loader validity check | Loader cancels keyed job | Shutdown/runtime diagnostics | Placement can rebuild later | +| Resource absent | Loader check | Loader cancels keyed job | Runtime diagnostics | Reload/requeue placement | +| Final placement release | Placement registry returns final | Loader cancels build then frees Node | Registry/shutdown regressions | Later placement starts fresh | +| Reset/shutdown | Loader lifecycle | Queue clear releases references | Shutdown verifier | Next map starts empty | + +## Configuration and capabilities + +The module introduces no settings or profile branches. WMO group permit count, +quality presets, cache limits, visibility and shadows remain loader-owned. + +## Persistence, cache and migration + +No data is serialized. WMO cache format/version/path and load request behavior +are unchanged. No migration or cache rebuild is required. + +## Diagnostics and observability + +`pending_count` preserves existing queue metrics, including stale/duplicate keys. +The detached snapshot exposes FIFO order, cursor values and root/resource presence +without exposing mutable jobs or engine objects. No runtime logs are emitted. + +## Verification + +- `verify_wmo_render_build_queue.gd`: invalid inputs, FIFO, typed references, + cursor adoption, duplicate replacement, stale pop, first-key cancel, clear, + engine lifetime, detached sorted diagnostics, source boundaries and timing. +- WMO planner/registry/resolver/material/shutdown and renderer regressions verify + unchanged ordering, permit use, destruction and surrounding lifecycle. +- Fidelity evidence is exact preservation of current queue/Dictionary semantics; + no original-client visual or asset-backed performance claim is made. + +## Extension points + +- Cache/load-request ownership can be extracted separately without changing jobs. +- A later typed materialization command may consume the borrowed job and planner + output while this queue remains the pending-state owner. + +## Capability status + +| Capability | Status | Evidence | Gap/next step | +|---|---|---|---| +| WMO render build pending-state ownership | Implemented | Lifecycle/source/timing verifier | Asset-backed traversal/leak evidence pending | +| WMO group planning | Implemented | Build-step planner verifier | Asset-backed p95/p99 pending | +| WMO materialization/cache loading | Partial | Existing runtime regressions | Separate loader seams remain | + +## Known gaps and risks + +- Jobs intentionally retain strong Node/Resource references until cancel/clear. +- Duplicate enqueue preserves a stale-key edge case rather than normalizing it. +- Resource array shape remains a WMO cache-builder invariant. +- No proprietary WMO corpus, traversal profile or original-client paired capture + is included in this package. + +## Source map + +| Path | Responsibility | +|---|---| +| `src/render/wmo/wmo_render_build_job.gd` | Typed references and cursors | +| `src/render/wmo/wmo_render_build_queue.gd` | Keyed jobs, FIFO order, cancel/clear and diagnostics | +| `src/render/wmo/wmo_render_build_step_planner.gd` | Pure operation/cursor planning | +| `src/scenes/streaming/streaming_world_loader.gd` | Validation, materialization, permits and engine destruction | +| `src/tools/verify_wmo_render_build_queue.gd` | Lifecycle, ownership, boundary and timing regression | + +## Related decisions and references + +- [`wmo-render-build-step-planner.md`](wmo-render-build-step-planner.md) +- [`wmo-placement-registry.md`](wmo-placement-registry.md) +- [`world-renderer.md`](world-renderer.md) +- [`../../RENDER.md`](../../RENDER.md) +- [`../../targets/roadmap/02-rendering-and-graphics.md`](../../targets/roadmap/02-rendering-and-graphics.md) diff --git a/docs/modules/wmo-render-build-step-planner.md b/docs/modules/wmo-render-build-step-planner.md index fdae2c9..3f363da 100644 --- a/docs/modules/wmo-render-build-step-planner.md +++ b/docs/modules/wmo-render-build-step-planner.md @@ -104,7 +104,8 @@ sequenceDiagram ## Ownership, threading and resources - The planner owns only call-local scalar comparisons and the returned value map. -- The loader owns Resources, mesh arrays, build jobs/queue and cursor adoption. +- `WmoRenderBuildQueue` owns typed jobs, FIFO keys and strong Node3D/Resource + references. The loader owns mesh-array interpretation and engine lifetime. - The loader creates and owns every MeshInstance3D/MultiMeshInstance3D and performs material refresh and editor-owner assignment on the main thread. - `RenderBudgetScheduler` owns permits; the planner neither reads nor consumes one. @@ -149,8 +150,8 @@ existing loader/facade diagnostics. ## Extension points -- WMO render build job/queue ownership may be extracted separately after a typed - cancellation and Resource lifetime contract exists. +- `WmoRenderBuildQueue` now owns typed pending state; cache/load requests and + materialization remain separate extraction seams. - Materialization can later receive typed commands without changing this planner. ## Capability status @@ -158,7 +159,7 @@ existing loader/facade diagnostics. | Capability | Status | Evidence | Gap/next step | |---|---|---|---| | WMO render build step planning | Implemented | Pure contract/source/timing verifier | Asset-backed traversal p95/p99 pending | -| WMO build job/resource ownership | Partial | Existing loader/runtime regressions | Separate state/cancellation package | +| WMO build job/queue ownership | Implemented | Typed lifecycle/source/timing verifier | Asset-backed traversal/leak evidence pending | | WMO visual/portal/material fidelity | Partial | Renderer baseline/material checks | Original-client paired evidence pending | ## Known gaps and risks @@ -173,12 +174,15 @@ existing loader/facade diagnostics. | Path | Responsibility | |---|---| | `src/render/wmo/wmo_render_build_step_planner.gd` | Pure operation and cursor selection | -| `src/scenes/streaming/streaming_world_loader.gd` | Resource validation, jobs/queues, materialization, permits and cancellation | +| `src/render/wmo/wmo_render_build_job.gd` | Typed engine references and cursors | +| `src/render/wmo/wmo_render_build_queue.gd` | Pending jobs, FIFO keys and cancellation state | +| `src/scenes/streaming/streaming_world_loader.gd` | Resource validation, materialization, permits and engine lifetime | | `src/tools/verify_wmo_render_build_step_planner.gd` | Contract, boundary and timing regression | ## Related decisions and references - [`wmo-placement-registry.md`](wmo-placement-registry.md) +- [`wmo-render-build-queue.md`](wmo-render-build-queue.md) - [`wmo-placement-resolver.md`](wmo-placement-resolver.md) - [`world-renderer.md`](world-renderer.md) - [`../../RENDER.md`](../../RENDER.md) diff --git a/docs/modules/world-renderer.md b/docs/modules/world-renderer.md index a77d9c5..664bbae 100644 --- a/docs/modules/world-renderer.md +++ b/docs/modules/world-renderer.md @@ -7,7 +7,7 @@ | Status | Partial | | Target/work package | M00 baseline; `M01-RND-STREAMING-FOCUS-001`; `M01-QAR-SERVER-SPAWN-RENDERER-001`; M03 facade/planner/scheduler/internal-access/ground/environment/entity packages; M03 terrain packages; M03 M2 packages; M03 WMO placement package | | Owners | Renderer workstream / milestone integrator | -| Last verified | Worktree `work/sindo-main-codex/m03-wmo-render-build-planner`, 2026-07-17 | +| Last verified | Worktree `work/sindo-main-codex/m03-wmo-render-build-queue`, 2026-07-17 | | Profiles/capabilities | `Performance`, `Balanced`, `High`, `Custom`; Blizzlike fidelity incomplete | ## Purpose @@ -60,6 +60,8 @@ flowchart LR WmoRegistry --> Loader Loader --> WmoBuildStep[WmoRenderBuildStepPlanner] WmoBuildStep --> Loader + Loader --> WmoBuildQueue[WmoRenderBuildQueue] + WmoBuildQueue --> Loader Native --> Parsed[Parsed tile/model data] Parsed --> Loader Loader --> Scene[SceneTree nodes] @@ -133,6 +135,7 @@ from externally reading/writing loader-private queue, task, cache and tile-state | `WmoPlacementResolver.normalize_relative_path/resolve_unique_key/resolve_world_transform` | Internal pure WMO service | Resolves cache key, registry identity and world transform | Main/any thread; stateless | Missing UID uses tile/index fallback; transform fields use historical defaults | | `WmoPlacementRegistry.add_reference/release_reference/contains/active_count/diagnostic_snapshot/clear` | Internal WMO service | Owns placement-key to tile/global reference sets | Renderer main thread; map session | Empty/unknown/non-owner input is rejected without mutation | | `WmoRenderBuildStepPlanner.plan_step` | Internal pure WMO service | Selects one mesh-first lightweight render-group operation and next cursors | Main/any thread; stateless | Raw integer comparisons are preserved without clamping | +| `WmoRenderBuildQueue` / `WmoRenderBuildJob` | Internal WMO pending-state service | Owns typed root/resource/cursor jobs and FIFO placement keys | Renderer main thread; map session | Invalid enqueue rejected; duplicate/stale behavior preserved | Публичным contract не считаются `StreamingWorldLoader` methods/properties, внутренние dictionaries, queues, job records и generated node names. Scene-owned @@ -164,6 +167,7 @@ loader configuration remains transitional composition data, not a caller API. | Internal WMO placement | Path, MODF placement, tile/index | Loader / `WmoPlacementResolver` | WMO caches, registry and three instance adapters | Value-only String/Transform3D | Lookup/placement lifetime | | Internal WMO ownership | Resolved placement key and tile/global reference key | Loader / `WmoPlacementRegistry` | Loader create/retain/final-free decisions | Registry-owned String sets; detached diagnostics | Map session or final release | | Internal WMO build step | Mesh/MultiMesh counts and job cursors | Loader / `WmoRenderBuildStepPlanner` | Loader materialization/cursor adapter | Fresh scalar Dictionary | One group operation | +| Internal WMO pending build | Placement key, Node3D root, WMO Resource and cursors | Loader / `WmoRenderBuildQueue` | Loader drain and step planner adapter | Queue-owned job and strong references | Until cancel/clear/replacement | | Output | Desired tile/detail operations | Streamer plan application | Finalize queues | Loader-owned | Cross-frame | | Output | Terrain/M2/WMO/liquid instances | Loader/builders | Godot world/renderer | Loader/world owner | Main-thread attach | | Output | Detached renderer metrics | `WorldRenderFacade` | Capture/baseline tools | Caller-owned deep copy | On demand | @@ -214,7 +218,8 @@ flowchart TD M2Batch --> M2 R --> WmoPlacement[WmoPlacementResolver] WmoPlacement --> WmoRegistry[WmoPlacementRegistry] - WmoRegistry --> WmoBuildStep[WmoRenderBuildStepPlanner] + WmoRegistry --> WmoBuildQueue[WmoRenderBuildQueue] + WmoBuildQueue --> WmoBuildStep[WmoRenderBuildStepPlanner] WmoBuildStep --> WMO R --> B[RenderBudgetScheduler permits] B --> Terrain[Terrain attach/upgrade] @@ -340,8 +345,10 @@ sequenceDiagram - `WmoPlacementResolver` is stateless and owns only call-local cache-key, identity and transform values. `WmoPlacementRegistry` owns only placement-key reference sets. `WmoRenderBuildStepPlanner` owns only a call-local operation - and cursor plan. The loader retains the key-to-Node map, caches, jobs, queues, - resources, cursor adoption, permits, cancellation and every Node lifecycle action. + and cursor plan. `WmoRenderBuildQueue` owns typed pending jobs, FIFO keys and + strong root/resource references without freeing engine objects. The loader + retains the placement key-to-Node map, caches/loads, materialization, permits, + validity reactions and every Node lifecycle action. - Rendered-ground query results and diagnostic snapshots are caller-owned values; the facade never returns Mesh, Node, tile-state or queue references. - Loaded-mesh ground sampling is renderer diagnostics, not authoritative gameplay @@ -439,6 +446,8 @@ Exact exported settings and cache versions remain documented in [`../../RENDER.m lifecycle, detached sorted diagnostics, source ownership and bounded timing. - WMO render build step contract: mesh-before-MultiMesh order, one-index cursor transition, completion/raw integer behavior, source ownership and bounded timing. +- WMO render build queue contract: typed references/cursors, FIFO, duplicate + replacement, stale-front cleanup, cancel/clear engine lifetime and bounded timing. - Budget scheduler contract: exact lane exhaustion, shared chunk removal/create priority, independent lanes, frame reset, invalid limits, terminal cancellation, dependency boundaries and bounded permit timing without loading a world scene. @@ -474,7 +483,8 @@ Exact exported settings and cache versions remain documented in [`../../RENDER.m | M2 build batch planner | Implemented extraction | Scene-free limit/count/cursor/source/timing contract | Queue/resource state, spatial cells and asset-backed p95/p99 remain pending | | WMO placement resolver | Implemented extraction | Scene-free path/identity/transform/source/timing contract | Asset-backed comparison pending | | WMO placement registry | Implemented extraction | Scene-free ownership/lifecycle/source/timing contract | Build/resource state and asset-backed cross-tile corpus pending | -| WMO render build step planner | Implemented extraction | Scene-free order/cursor/source/timing contract | Job/resource ownership and asset-backed traversal p95/p99 pending | +| WMO render build step planner | Implemented extraction | Scene-free order/cursor/source/timing contract | Asset-backed traversal p95/p99 pending | +| WMO render build queue | Implemented extraction | Typed lifecycle/order/ownership/source/timing contract | Cache/load state and asset-backed traversal/leak evidence pending | | WMO rendering | Partial | Cached group rendering | Portals/rooms/material parity | | Liquids | Partial | MH2O/MLIQ paths | LiquidType/depth/shore fidelity | | Sky/light | Partial | DBC controller/logs | Zone transition paired validation | diff --git a/src/render/wmo/wmo_render_build_job.gd b/src/render/wmo/wmo_render_build_job.gd new file mode 100644 index 0000000..ec800f2 --- /dev/null +++ b/src/render/wmo/wmo_render_build_job.gd @@ -0,0 +1,59 @@ +class_name WmoRenderBuildJob +extends RefCounted + +## Holds one lightweight-WMO build cursor and strong engine-object references. +## Releasing this record never frees the referenced Node or Resource. + +var _placement_key: String +var _root: Node3D +var _render_resource: Resource +var _mesh_index: int = 0 +var _multimesh_index: int = 0 + + +func _init(placement_key: String, root: Node3D, render_resource: Resource) -> void: + _placement_key = placement_key + _root = root + _render_resource = render_resource + + +## Returns the immutable placement identity used by the queue. +func placement_key() -> String: + return _placement_key + + +## Returns the strongly referenced scene root without transferring ownership. +func root() -> Node3D: + return _root + + +## Returns the strongly referenced render resource without transferring ownership. +func render_resource() -> Resource: + return _render_resource + + +## Returns the current mesh-group cursor. +func mesh_index() -> int: + return _mesh_index + + +## Returns the current doodad MultiMesh-group cursor. +func multimesh_index() -> int: + return _multimesh_index + + +## Atomically adopts both cursor values returned by the pure step planner. +func adopt_cursors(next_mesh_index: int, next_multimesh_index: int) -> void: + _mesh_index = next_mesh_index + _multimesh_index = next_multimesh_index + + +## Returns detached scalar state and never exposes engine-object references. +func diagnostic_snapshot() -> Dictionary: + return { + "placement_key": _placement_key, + "mesh_index": _mesh_index, + "multimesh_index": _multimesh_index, + "has_root": _root != null, + "has_render_resource": _render_resource != null, + } diff --git a/src/render/wmo/wmo_render_build_job.gd.uid b/src/render/wmo/wmo_render_build_job.gd.uid new file mode 100644 index 0000000..0f26f95 --- /dev/null +++ b/src/render/wmo/wmo_render_build_job.gd.uid @@ -0,0 +1 @@ +uid://gyjq5dqueyi7 diff --git a/src/render/wmo/wmo_render_build_queue.gd b/src/render/wmo/wmo_render_build_queue.gd new file mode 100644 index 0000000..f902cd5 --- /dev/null +++ b/src/render/wmo/wmo_render_build_queue.gd @@ -0,0 +1,88 @@ +class_name WmoRenderBuildQueue +extends RefCounted + +## Owns pending lightweight-WMO job records and FIFO placement keys. + +const JOB_SCRIPT := preload("res://src/render/wmo/wmo_render_build_job.gd") + +var _jobs_by_placement_key: Dictionary = {} +var _queued_placement_keys: Array[String] = [] + + +## Enqueues a new typed job. Duplicate placement keys replace the keyed job but +## append another FIFO entry, preserving the loader's historical behavior. +func enqueue(placement_key: String, root: Node3D, render_resource: Resource) -> bool: + if placement_key.is_empty() or root == null or render_resource == null: + return false + _jobs_by_placement_key[placement_key] = JOB_SCRIPT.new( + placement_key, + root, + render_resource + ) + _queued_placement_keys.append(placement_key) + return true + + +## Returns whether at least one FIFO key, including a stale key, is pending. +func has_pending() -> bool: + return not _queued_placement_keys.is_empty() + + +## Returns the front FIFO key, or an empty String when the queue is empty. +func front_key() -> String: + if _queued_placement_keys.is_empty(): + return "" + return _queued_placement_keys.front() + + +## Returns the current typed job for a placement key, or null for a stale key. +func job_for(placement_key: String) -> WmoRenderBuildJob: + return _jobs_by_placement_key.get(placement_key, null) as WmoRenderBuildJob + + +## Pops and returns the front FIFO key, or an empty String when already empty. +func pop_front() -> String: + if _queued_placement_keys.is_empty(): + return "" + return _queued_placement_keys.pop_front() + + +## Cancels the keyed job and erases only the first matching FIFO key. Returns +## whether an active job record existed. Engine objects are never freed here. +func cancel(placement_key: String) -> bool: + var had_job := _jobs_by_placement_key.has(placement_key) + _jobs_by_placement_key.erase(placement_key) + _queued_placement_keys.erase(placement_key) + return had_job + + +## Releases all job/key references without freeing referenced engine objects. +func clear() -> void: + _jobs_by_placement_key.clear() + _queued_placement_keys.clear() + + +## Returns FIFO entry count, including duplicate and stale keys. +func pending_count() -> int: + return _queued_placement_keys.size() + + +## Returns active keyed job count, excluding stale FIFO keys. +func active_job_count() -> int: + return _jobs_by_placement_key.size() + + +## Returns detached FIFO order and placement-key-sorted scalar job diagnostics. +func diagnostic_snapshot() -> Dictionary: + var placement_keys: Array = _jobs_by_placement_key.keys() + placement_keys.sort() + var jobs: Array[Dictionary] = [] + for placement_key_variant in placement_keys: + var placement_key := String(placement_key_variant) + var job := job_for(placement_key) + if job != null: + jobs.append(job.diagnostic_snapshot()) + return { + "queued_placement_keys": _queued_placement_keys.duplicate(), + "jobs": jobs, + } diff --git a/src/render/wmo/wmo_render_build_queue.gd.uid b/src/render/wmo/wmo_render_build_queue.gd.uid new file mode 100644 index 0000000..e255697 --- /dev/null +++ b/src/render/wmo/wmo_render_build_queue.gd.uid @@ -0,0 +1 @@ +uid://b165sgg76lrt8 diff --git a/src/scenes/streaming/streaming_world_loader.gd b/src/scenes/streaming/streaming_world_loader.gd index 00a2173..02ed54b 100644 --- a/src/scenes/streaming/streaming_world_loader.gd +++ b/src/scenes/streaming/streaming_world_loader.gd @@ -18,6 +18,9 @@ const WMO_PLACEMENT_REGISTRY_SCRIPT := preload( const WMO_RENDER_BUILD_STEP_PLANNER_SCRIPT := preload( "res://src/render/wmo/wmo_render_build_step_planner.gd" ) +const WMO_RENDER_BUILD_QUEUE_SCRIPT := preload( + "res://src/render/wmo/wmo_render_build_queue.gd" +) const M2_BUILDER_SCRIPT := preload("res://addons/mpq_extractor/loaders/m2_builder.gd") const M2_NATIVE_ANIMATED_BUILDER_SCRIPT := preload("res://addons/mpq_extractor/loaders/m2_native_animated_builder.gd") const M2_NATIVE_ANIMATOR_SCRIPT := preload("res://src/scenes/streaming/m2_native_animator.gd") @@ -280,8 +283,7 @@ var _wmo_prototype_cache: Dictionary = {} var _wmo_render_cache: Dictionary = {} var _wmo_render_missing_cache: Dictionary = {} var _wmo_render_load_requests: Dictionary = {} -var _wmo_render_build_jobs: Dictionary = {} -var _wmo_render_build_queue: Array = [] +var _wmo_render_build_queue := WMO_RENDER_BUILD_QUEUE_SCRIPT.new() var _wmo_scene_resource_cache: Dictionary = {} var _wmo_scene_cache_missing: Dictionary = {} var _wmo_scene_load_requests: Dictionary = {} @@ -1004,7 +1006,7 @@ func _log_hitch_profile(start_usec: int, timings: Array[String], did_refresh: bo _m2_mesh_finalize_queue.size() + _m2_mesh_load_requests.size(), _m2_build_queue.size(), _wmo_build_queue.size() + _wmo_scene_load_requests.size() + _wmo_render_load_requests.size(), - _wmo_render_build_queue.size(), + _wmo_render_build_queue.pending_count(), _tile_lod_create_queue.size(), _tile_lod_remove_queue.size(), _chunk_create_queue.size(), @@ -1031,7 +1033,7 @@ func render_baseline_snapshot() -> Dictionary: "m2_mesh": _m2_mesh_finalize_queue.size() + _m2_mesh_load_requests.size(), "m2_build": _m2_build_queue.size(), "wmo_build": _wmo_build_queue.size() + _wmo_scene_load_requests.size() + _wmo_render_load_requests.size(), - "wmo_groups": _wmo_render_build_queue.size(), + "wmo_groups": _wmo_render_build_queue.pending_count(), "lod_create": _tile_lod_create_queue.size(), "lod_remove": _tile_lod_remove_queue.size(), "chunk_create": _chunk_create_queue.size(), @@ -1313,7 +1315,7 @@ func _tick_runtime_stats(delta: float) -> void: _m2_mesh_finalize_queue.size() + _m2_mesh_load_requests.size(), _m2_build_queue.size(), _wmo_build_queue.size() + _wmo_scene_load_requests.size() + _wmo_render_load_requests.size(), - _wmo_render_build_queue.size(), + _wmo_render_build_queue.pending_count(), _tile_lod_create_queue.size(), _tile_lod_remove_queue.size(), _chunk_create_queue.size(), @@ -2364,7 +2366,6 @@ func _wait_for_tile_tasks() -> void: _wmo_render_load_requests.clear() _wmo_render_missing_cache.clear() _wmo_scene_cache_missing.clear() - _wmo_render_build_jobs.clear() _wmo_render_build_queue.clear() for pending in _terrain_upgrade_tasks.values(): @@ -3004,7 +3005,6 @@ func _clear_streamed_world() -> void: _wmo_render_load_requests.clear() _wmo_render_missing_cache.clear() _wmo_scene_cache_missing.clear() - _wmo_render_build_jobs.clear() _wmo_render_build_queue.clear() _tile_result_mutex.lock() @@ -3761,32 +3761,24 @@ func _instantiate_wmo_render_root(rel_path: String, placement: Dictionary) -> No func _start_wmo_render_build(unique_key: String, root: Node3D, render_resource: Resource) -> void: - if unique_key.is_empty() or root == null or render_resource == null: - return - _wmo_render_build_jobs[unique_key] = { - "root": root, - "resource": render_resource, - "mesh_index": 0, - "multimesh_index": 0, - } - _wmo_render_build_queue.append(unique_key) + _wmo_render_build_queue.enqueue(unique_key, root, render_resource) func _process_wmo_render_build_jobs() -> void: while _render_budget_scheduler.has_remaining_permit( - RENDER_BUDGET_SCHEDULER_SCRIPT.WMO_RENDER_GROUP_BUILD) and not _wmo_render_build_queue.is_empty(): - var unique_key := String(_wmo_render_build_queue.front()) - if not _wmo_render_build_jobs.has(unique_key): + RENDER_BUDGET_SCHEDULER_SCRIPT.WMO_RENDER_GROUP_BUILD) and _wmo_render_build_queue.has_pending(): + var unique_key := _wmo_render_build_queue.front_key() + var job := _wmo_render_build_queue.job_for(unique_key) + if job == null: _wmo_render_build_queue.pop_front() continue - var job: Dictionary = _wmo_render_build_jobs[unique_key] - var root: Node = job.get("root", null) + var root: Node = job.root() if root == null or not is_instance_valid(root): _cancel_wmo_render_build_job(unique_key) continue - var render_resource: Resource = job.get("resource", null) + var render_resource: Resource = job.render_resource() if render_resource == null: _cancel_wmo_render_build_job(unique_key) continue @@ -3794,11 +3786,11 @@ func _process_wmo_render_build_jobs() -> void: var meshes: Array = render_resource.get("meshes") var mesh_transforms: Array = render_resource.get("mesh_transforms") var mesh_names: PackedStringArray = render_resource.get("mesh_names") - var mesh_index: int = int(job.get("mesh_index", 0)) + var mesh_index := job.mesh_index() var multimeshes: Array = render_resource.get("multimeshes") var multimesh_transforms: Array = render_resource.get("multimesh_transforms") var multimesh_names: PackedStringArray = render_resource.get("multimesh_names") - var multimesh_index: int = int(job.get("multimesh_index", 0)) + var multimesh_index := job.multimesh_index() var build_step: Dictionary = _wmo_render_build_step_planner.plan_step( meshes.size(), mesh_index, @@ -3829,8 +3821,10 @@ func _process_wmo_render_build_jobs() -> void: mesh_instance.visibility_range_end_margin = CHUNK_SIZE (root as Node3D).add_child(mesh_instance) _set_editor_owner_recursive(mesh_instance) - job["mesh_index"] = int(build_step["next_mesh_index"]) - _wmo_render_build_jobs[unique_key] = job + job.adopt_cursors( + int(build_step["next_mesh_index"]), + int(build_step["next_multimesh_index"]) + ) _render_budget_scheduler.try_consume_permit(RENDER_BUDGET_SCHEDULER_SCRIPT.WMO_RENDER_GROUP_BUILD) continue @@ -3857,8 +3851,10 @@ func _process_wmo_render_build_jobs() -> void: multimesh_instance.visibility_range_end_margin = CHUNK_SIZE (root as Node3D).add_child(multimesh_instance) _set_editor_owner_recursive(multimesh_instance) - job["multimesh_index"] = int(build_step["next_multimesh_index"]) - _wmo_render_build_jobs[unique_key] = job + job.adopt_cursors( + int(build_step["next_mesh_index"]), + int(build_step["next_multimesh_index"]) + ) _render_budget_scheduler.try_consume_permit(RENDER_BUDGET_SCHEDULER_SCRIPT.WMO_RENDER_GROUP_BUILD) continue @@ -3866,9 +3862,7 @@ func _process_wmo_render_build_jobs() -> void: func _cancel_wmo_render_build_job(unique_key: String) -> void: - if _wmo_render_build_jobs.has(unique_key): - _wmo_render_build_jobs.erase(unique_key) - _wmo_render_build_queue.erase(unique_key) + _wmo_render_build_queue.cancel(unique_key) func _get_resource_file_size(path: String) -> int: @@ -4043,7 +4037,6 @@ func _unregister_tile_wmos(state: Dictionary) -> void: func _clear_wmo_registry() -> void: - _wmo_render_build_jobs.clear() _wmo_render_build_queue.clear() for node_variant in _wmo_instance_nodes_by_unique_key.values(): if not (node_variant is Node): diff --git a/src/tools/verify_wmo_render_build_queue.gd b/src/tools/verify_wmo_render_build_queue.gd new file mode 100644 index 0000000..849b436 --- /dev/null +++ b/src/tools/verify_wmo_render_build_queue.gd @@ -0,0 +1,202 @@ +extends SceneTree + +## Asset-free lifecycle, ordering, ownership and timing regression for WMO jobs. + +const QUEUE_SCRIPT := preload("res://src/render/wmo/wmo_render_build_queue.gd") +const QUEUE_PATH := "res://src/render/wmo/wmo_render_build_queue.gd" +const JOB_PATH := "res://src/render/wmo/wmo_render_build_job.gd" +const LOADER_PATH := "res://src/scenes/streaming/streaming_world_loader.gd" + + +func _initialize() -> void: + var failures: Array[String] = [] + _verify_invalid_and_fifo_jobs(failures) + _verify_cursor_adoption(failures) + _verify_duplicate_and_stale_key_behavior(failures) + _verify_cancel_clear_and_engine_lifetime(failures) + _verify_detached_sorted_diagnostics(failures) + _verify_loader_and_engine_lifetime_boundaries(failures) + var elapsed_milliseconds := _verify_bounded_timing(failures) + if not failures.is_empty(): + for failure in failures: + push_error("WMO_RENDER_BUILD_QUEUE: %s" % failure) + quit(1) + return + print( + "WMO_RENDER_BUILD_QUEUE PASS cases=11 iterations=100 elapsed_ms=%.3f" + % elapsed_milliseconds + ) + quit(0) + + +func _verify_invalid_and_fifo_jobs(failures: Array[String]) -> void: + var queue: RefCounted = QUEUE_SCRIPT.new() + var root := Node3D.new() + var render_resource := Resource.new() + _expect_false(queue.call("enqueue", "", root, render_resource), "empty key rejected", failures) + _expect_false(queue.call("enqueue", "uid:1", null, render_resource), "null root rejected", failures) + _expect_false(queue.call("enqueue", "uid:1", root, null), "null resource rejected", failures) + _expect_true(queue.call("enqueue", "uid:2", root, render_resource), "first enqueue", failures) + _expect_true(queue.call("enqueue", "uid:3", root, render_resource), "second enqueue", failures) + _expect_string_equal(String(queue.call("front_key")), "uid:2", "FIFO front", failures) + _expect_equal(int(queue.call("pending_count")), 2, "FIFO pending count", failures) + _expect_equal(int(queue.call("active_job_count")), 2, "FIFO active count", failures) + var first_job: RefCounted = queue.call("job_for", "uid:2") + _expect_string_equal(String(first_job.call("placement_key")), "uid:2", "typed job key", failures) + _expect_true(first_job.call("root") == root, "job retains root", failures) + _expect_true(first_job.call("render_resource") == render_resource, "job retains resource", failures) + queue.call("clear") + root.free() + + +func _verify_cursor_adoption(failures: Array[String]) -> void: + var queue: RefCounted = QUEUE_SCRIPT.new() + var root := Node3D.new() + queue.call("enqueue", "uid:4", root, Resource.new()) + var job: RefCounted = queue.call("job_for", "uid:4") + _expect_equal(int(job.call("mesh_index")), 0, "initial mesh cursor", failures) + _expect_equal(int(job.call("multimesh_index")), 0, "initial multimesh cursor", failures) + job.call("adopt_cursors", 7, 3) + _expect_equal(int(job.call("mesh_index")), 7, "adopted mesh cursor", failures) + _expect_equal(int(job.call("multimesh_index")), 3, "adopted multimesh cursor", failures) + queue.call("clear") + root.free() + + +func _verify_duplicate_and_stale_key_behavior(failures: Array[String]) -> void: + var queue: RefCounted = QUEUE_SCRIPT.new() + var first_root := Node3D.new() + var second_root := Node3D.new() + queue.call("enqueue", "uid:5", first_root, Resource.new()) + queue.call("enqueue", "uid:5", second_root, Resource.new()) + _expect_equal(int(queue.call("pending_count")), 2, "duplicate keys append", failures) + _expect_equal(int(queue.call("active_job_count")), 1, "duplicate job replaces", failures) + var replacement_job: RefCounted = queue.call("job_for", "uid:5") + _expect_true(replacement_job.call("root") == second_root, "duplicate latest job wins", failures) + _expect_true(queue.call("cancel", "uid:5"), "duplicate cancel finds job", failures) + _expect_equal(int(queue.call("pending_count")), 1, "cancel erases first duplicate key", failures) + _expect_true(queue.call("job_for", "uid:5") == null, "remaining duplicate key is stale", failures) + _expect_string_equal(String(queue.call("pop_front")), "uid:5", "stale front pop", failures) + _expect_false(queue.call("has_pending"), "stale queue drained", failures) + first_root.free() + second_root.free() + + +func _verify_cancel_clear_and_engine_lifetime(failures: Array[String]) -> void: + var queue: RefCounted = QUEUE_SCRIPT.new() + var root := Node3D.new() + queue.call("enqueue", "uid:6", root, Resource.new()) + _expect_false(queue.call("cancel", "uid:missing"), "unknown cancel", failures) + _expect_true(is_instance_valid(root), "unknown cancel keeps root valid", failures) + queue.call("clear") + queue.call("clear") + _expect_equal(int(queue.call("pending_count")), 0, "clear idempotent", failures) + _expect_true(is_instance_valid(root), "clear does not free root", failures) + root.free() + + +func _verify_detached_sorted_diagnostics(failures: Array[String]) -> void: + var queue: RefCounted = QUEUE_SCRIPT.new() + var root := Node3D.new() + queue.call("enqueue", "uid:20", root, Resource.new()) + queue.call("enqueue", "uid:10", root, Resource.new()) + var snapshot: Dictionary = queue.call("diagnostic_snapshot") + _expect_string_array( + snapshot["queued_placement_keys"], + ["uid:20", "uid:10"], + "diagnostic queue order", + failures + ) + var jobs: Array = snapshot["jobs"] + _expect_string_equal(String(jobs[0]["placement_key"]), "uid:10", "diagnostic job sort", failures) + (snapshot["queued_placement_keys"] as Array).append("mutated") + jobs[0]["mesh_index"] = 99 + var fresh_snapshot: Dictionary = queue.call("diagnostic_snapshot") + _expect_string_array( + fresh_snapshot["queued_placement_keys"], + ["uid:20", "uid:10"], + "diagnostic queue detached", + failures + ) + _expect_equal(int((fresh_snapshot["jobs"] as Array)[0]["mesh_index"]), 0, "job diagnostic detached", failures) + queue.call("clear") + root.free() + + +func _verify_loader_and_engine_lifetime_boundaries(failures: Array[String]) -> void: + var loader_source := _read_text(LOADER_PATH, failures) + var queue_source := _read_text(QUEUE_PATH, failures) + var job_source := _read_text(JOB_PATH, failures) + _expect_true( + loader_source.contains("WMO_RENDER_BUILD_QUEUE_SCRIPT.new()"), + "loader composes queue", + failures + ) + _expect_true(loader_source.contains("_wmo_render_build_queue.enqueue("), "loader delegates enqueue", failures) + _expect_true(loader_source.contains("_wmo_render_build_queue.job_for("), "loader reads typed job", failures) + _expect_true(loader_source.contains("_wmo_render_build_queue.cancel("), "loader delegates cancel", failures) + _expect_true(loader_source.contains("_wmo_render_build_queue.clear()"), "loader delegates clear", failures) + _expect_false(loader_source.contains("var _wmo_render_build_jobs"), "legacy job dictionary removed", failures) + _expect_true( + loader_source.contains("MeshInstance3D.new()") + and loader_source.contains("MultiMeshInstance3D.new()") + and loader_source.contains("try_consume_permit("), + "loader retains materialization and permits", + failures + ) + for forbidden_token in ["queue_free", "RenderingServer", "RID", "MeshInstance3D", "MultiMeshInstance3D", "FileAccess", "Thread", "Mutex"]: + _expect_false( + queue_source.contains(forbidden_token) or job_source.contains(forbidden_token), + "queue/job omit %s ownership" % forbidden_token, + failures + ) + + +func _verify_bounded_timing(failures: Array[String]) -> float: + var queue: RefCounted = QUEUE_SCRIPT.new() + var root := Node3D.new() + var render_resource := Resource.new() + var started_microseconds := Time.get_ticks_usec() + for _iteration in range(100): + for job_index in range(256): + queue.call("enqueue", "uid:%d" % job_index, root, render_resource) + for job_index in range(256): + queue.call("cancel", "uid:%d" % job_index) + _expect_equal(int(queue.call("pending_count")), 0, "timing cycle drains", failures) + var elapsed_milliseconds := float(Time.get_ticks_usec() - started_microseconds) / 1000.0 + _expect_true(elapsed_milliseconds < 1000.0, "100 queue cycles remain bounded", failures) + root.free() + return elapsed_milliseconds + + +func _read_text(path: String, failures: Array[String]) -> String: + var file := FileAccess.open(path, FileAccess.READ) + if file == null: + failures.append("cannot open %s" % path) + return "" + return file.get_as_text() + + +func _expect_equal(actual_value: int, expected_value: int, label: String, failures: Array[String]) -> void: + if actual_value != expected_value: + failures.append("%s expected %d, got %d" % [label, expected_value, actual_value]) + + +func _expect_string_equal(actual_value: String, expected_value: String, label: String, failures: Array[String]) -> void: + if actual_value != expected_value: + failures.append("%s expected %s, got %s" % [label, expected_value, actual_value]) + + +func _expect_string_array(actual_values: Array, expected_values: Array, label: String, failures: Array[String]) -> void: + if actual_values != expected_values: + failures.append("%s expected %s, got %s" % [label, expected_values, actual_values]) + + +func _expect_true(actual_value: bool, label: String, failures: Array[String]) -> void: + if not actual_value: + failures.append("%s expected true" % label) + + +func _expect_false(actual_value: bool, label: String, failures: Array[String]) -> void: + if actual_value: + failures.append("%s expected false" % label) diff --git a/src/tools/verify_wmo_render_build_queue.gd.uid b/src/tools/verify_wmo_render_build_queue.gd.uid new file mode 100644 index 0000000..50c7d8d --- /dev/null +++ b/src/tools/verify_wmo_render_build_queue.gd.uid @@ -0,0 +1 @@ +uid://s6bsyt6bv1uq