From a984e705c3e8da68d173e99db49b28dac0f8266f Mon Sep 17 00:00:00 2001 From: sindoring Date: Sun, 16 Aug 2026 16:56:34 +0400 Subject: [PATCH] =?UTF-8?q?=D0=BE=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D1=81=D0=B1=D0=BE=D1=80=D1=89=D0=B8=D0=BA?= =?UTF-8?q?=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + .gitmodules | 3 +- CVPATCH_COMPATIBILITY.md | 17 +-- README.md | 5 + build-warcraftxl.ps1 | 73 ++++++++--- build_manifest.py | 4 + deploy.ps1 | 37 +++++- patch-layout.json | 26 ++++ patch_layout.py | 36 ++++++ pipeline-layout.ps1 | 241 +++++++++++++++++++++++++++++++++++ run.ps1 | 31 ++++- tool/src/main.rs | 31 ----- upload_to_s3.py | 183 +++++++++++++++++++++++--- validate_package.py | 135 ++++++++++++++++++++ vendor/modules/wxl-db2 | 2 +- vendor/modules/wxl-modern-m2 | 2 +- vendor/warcraftxl | 2 +- 17 files changed, 743 insertions(+), 86 deletions(-) create mode 100644 patch-layout.json create mode 100644 patch_layout.py create mode 100644 pipeline-layout.ps1 create mode 100644 validate_package.py diff --git a/.gitignore b/.gitignore index a023f99..1bad363 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ manifest.json .vscode dist build/ +/new customization mpqs/*.mpq Wow*.exe *.backup.exe Logs/ diff --git a/.gitmodules b/.gitmodules index 09ab9b4..a02daad 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,7 +15,8 @@ url = https://github.com/WarcraftXL/wxl-db2.git [submodule "vendor/modules/wxl-modern-m2"] path = vendor/modules/wxl-modern-m2 - url = https://github.com/WarcraftXL/wxl-modern-m2.git + url = https://github.com/sindoring/wxl-modern-m2.git + branch = agent/guard-oversized-m2-batches [submodule "vendor/modules/wxl-modern-wmo"] path = vendor/modules/wxl-modern-wmo url = https://github.com/WarcraftXL/wxl-modern-wmo.git diff --git a/CVPATCH_COMPATIBILITY.md b/CVPATCH_COMPATIBILITY.md index 3825c7e..5d484bd 100644 --- a/CVPATCH_COMPATIBILITY.md +++ b/CVPATCH_COMPATIBILITY.md @@ -283,8 +283,7 @@ build/customization-compat/server-dbc/BarberShopStyle.dbc Упаковщик: - собирает Z из исходников и сгенерированных MoonWell DBC; -- копирует `patch-a-001.mpq` в `dist\Data\ruRU\patch-ruRU-X.MPQ`; -- копирует `patch-b-002.mpq` в `dist\Data\ruRU\patch-ruRU-Y.MPQ`; +- не копирует внешние графические X/Y в `dist` и не включает их в managed manifest; - не добавляет `patch-k.mpq`. Для полной локальной сборки и синхронизации клиента можно использовать: @@ -303,13 +302,15 @@ build/customization-compat/server-dbc/BarberShopStyle.dbc ## Исходники MPQ и установленные копии -Файлы в `new customization mpqs` — источники сборки. В готовом клиенте -используются их копии X и Y. +Файлы в `new customization mpqs` используются только как локальные источники +генератора таблиц совместимости. В готовом клиенте X и Y управляются отдельно от +репозиторных патчей. -Технически уже установленный клиент может запускаться без исходников в -репозитории, если X/Y остаются в `Data\ruRU`. Однако удаление исходников сломает -повторную сборку `dist`, восстановление клиента и генерацию совместимости. -Поэтому `patch-a-001.mpq` и `patch-b-002.mpq` нужно хранить. +Списки репозиторных и внешних графических патчей находятся в +`patch-layout.json`. `deploy.ps1`, `run.ps1`, генератор манифеста и S3 uploader +синхронизируют только список `repositoryPatches`. Пути из `graphicsPatches` +сохраняются без копирования и удаления. Если один путь указан в обоих списках, +`repositoryPatches` имеет приоритет. Z не консолидирует многогигабайтные модели и текстуры X/Y. Удаление X или Y из клиента не освобождается наличием текущих DBC в Z. diff --git a/README.md b/README.md index 62ce41d..03fe75b 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,11 @@ WarcraftXL не требуется и в объектное хранилище - `patch-ruRU-5` — русская локализация и интерфейс; - `patch-Z` — Mythic+ ресурсы и клиентские данные. +Управляемые репозиторием MPQ и внешние графические патчи перечислены отдельно в +`patch-layout.json`. Сборка и deploy синхронизируют только `repositoryPatches` и +не изменяют `graphicsPatches`. При пересечении списков репозиторный патч имеет +приоритет. + ## Лицензирование WarcraftXL распространяется по GPL-3.0 и подключён как отдельный неизменённый upstream submodule. diff --git a/build-warcraftxl.ps1 b/build-warcraftxl.ps1 index d47e2c7..0202766 100644 --- a/build-warcraftxl.ps1 +++ b/build-warcraftxl.ps1 @@ -75,8 +75,53 @@ function Enable-LargeAddressAware { } } +function Initialize-MissingSubmodules { + param( + [Parameter(Mandatory)] + [string]$RepositoryPath + ) + + $modulesFile = Join-Path $RepositoryPath '.gitmodules' + if (-not (Test-Path -LiteralPath $modulesFile -PathType Leaf)) { + return + } + + $configuredModules = @( + & git -C $RepositoryPath config --file .gitmodules --get-regexp '^submodule\..*\.path$' + ) + if ($LASTEXITCODE -gt 1) { + throw "Failed to read submodules from $modulesFile" + } + + foreach ($configuredModule in $configuredModules) { + $fields = $configuredModule -split '\s+', 2 + if ($fields.Count -ne 2) { + continue + } + + $relativePath = $fields[1].Trim() + $modulePath = Join-Path $RepositoryPath $relativePath + $gitMarker = Join-Path $modulePath '.git' + + # Updating an initialized submodule checks out the SHA recorded by its parent and detaches + # any active development branch. Only initialize genuinely missing worktrees. + if (-not (Test-Path -LiteralPath $gitMarker)) { + Write-Host "Initializing missing submodule: $modulePath" + & git -C $RepositoryPath submodule update --init -- $relativePath + if ($LASTEXITCODE -ne 0) { + throw "Failed to initialize submodule: $modulePath" + } + } + + if (Test-Path -LiteralPath $modulePath -PathType Container) { + Initialize-MissingSubmodules -RepositoryPath $modulePath + } + } +} + $ErrorActionPreference = 'Stop' $repoRoot = $PSScriptRoot +. (Join-Path $repoRoot 'pipeline-layout.ps1') $cmake = 'C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe' if (-not (Test-Path -LiteralPath $cmake)) { $cmakeCommand = Get-Command cmake -ErrorAction SilentlyContinue @@ -84,8 +129,7 @@ if (-not (Test-Path -LiteralPath $cmake)) { $cmake = $cmakeCommand.Source } -& git -C $repoRoot submodule update --init --recursive -if ($LASTEXITCODE -ne 0) { throw 'Failed to initialize WarcraftXL submodules.' } +Initialize-MissingSubmodules -RepositoryPath $repoRoot $win32BuildDir = Join-Path $repoRoot 'build\warcraftxl-win32' $stockExe = Join-Path $repoRoot 'Wow_Original.exe' @@ -110,18 +154,7 @@ $win32ArtifactDir = Join-Path $win32BuildDir "vendor\warcraftxl\$Configuration" $warcraftXL = Join-Path $win32ArtifactDir 'WarcraftXL.dll' $selectedProxy = Join-Path $win32ArtifactDir 'd3d9.dll' $recoveryProxy = Join-Path $win32BuildDir "artifacts\$Configuration\d3d9-native.dll" -$extensionNames = @( - 'MoonWell', - 'wxl-db2', - 'wxl-fdid-moonwell', - 'wxl-grasswind', - 'wxl-modern-adt', - 'wxl-modern-blp', - 'wxl-modern-m2', - 'wxl-modern-wmo', - 'wxl-moonwell-storage-fallback', - 'wxl-unit-outline' -) +$extensionNames = Get-MoonWellExtensionNames if ($NativeRenderer) { Write-Warning '-NativeRenderer is no longer needed: WarcraftXL 1.1 uses native D3D9 by default.' @@ -133,6 +166,8 @@ function Install-WarcraftXLArtifacts { [string]$Destination ) + Remove-MoonWellRuntimeGarbage -Destination $Destination + $utils = Join-Path $Destination 'Utils' New-Item ` -ItemType Directory ` @@ -140,12 +175,6 @@ function Install-WarcraftXLArtifacts { -Force | Out-Null - $legacyHost = Join-Path $utils 'WarcraftXLHost.exe' - if (Test-Path -LiteralPath $legacyHost -PathType Leaf) { - Remove-Item -LiteralPath $legacyHost -Force - Write-Host "Removed obsolete WarcraftXL 1.0 host: $legacyHost" - } - $destinationExe = Join-Path $Destination 'Wow.exe' Copy-Item ` @@ -208,6 +237,10 @@ if ($Deploy) { if (-not [string]::IsNullOrWhiteSpace($PackagePath)) { $PackagePath = [System.IO.Path]::GetFullPath($PackagePath) + Assert-SafePackagePath ` + -PackagePath $PackagePath ` + -RepositoryPath $repoRoot ` + -ClientPath $ClientPath Install-WarcraftXLArtifacts -Destination $PackagePath Write-Host "Packaged stock Wow.exe and WarcraftXL artifacts in $PackagePath" } diff --git a/build_manifest.py b/build_manifest.py index a93c889..0db138a 100644 --- a/build_manifest.py +++ b/build_manifest.py @@ -9,6 +9,8 @@ import sys from pathlib import Path from typing import Optional +from patch_layout import PATCH_LAYOUT + IGNORED_TOP_LEVEL_DIRS = { "Cache", @@ -49,6 +51,8 @@ def is_ignored(rel_path: Path) -> bool: return True if len(rel_path.parts) == 1 and rel_path.name in IGNORED_TOP_LEVEL_FILES: return True + if PATCH_LAYOUT.is_external_graphics_patch(rel_path.as_posix()): + return True return any(part in IGNORED_DIRS_ANYWHERE for part in rel_path.parts[:-1]) diff --git a/deploy.ps1 b/deploy.ps1 index c98b7bc..db61662 100644 --- a/deploy.ps1 +++ b/deploy.ps1 @@ -13,6 +13,7 @@ param( $ErrorActionPreference = 'Stop' $repoRoot = $PSScriptRoot +. (Join-Path $repoRoot 'pipeline-layout.ps1') function Import-ProjectEnvironment { $envFile = Join-Path $repoRoot '.env' @@ -46,6 +47,10 @@ $PackagePath = [System.IO.Path]::GetFullPath($PackagePath) if (-not (Test-Path -LiteralPath $ClientPath -PathType Container)) { throw "Client directory was not found: $ClientPath" } +Assert-SafePackagePath ` + -PackagePath $PackagePath ` + -RepositoryPath $repoRoot ` + -ClientPath $ClientPath $runningClients = @(Get-Process Wow -ErrorAction SilentlyContinue | Where-Object { $_.Path -and $_.Path.StartsWith($ClientPath, [System.StringComparison]::OrdinalIgnoreCase) @@ -64,11 +69,21 @@ $env:WOW_HOME = $ClientPath Write-Host "MoonWell client: $ClientPath" Write-Host "Package staging: $PackagePath" -Write-Host '[1/5] Initializing WarcraftXL dependencies...' -& git -C $repoRoot submodule update --init --recursive -if ($LASTEXITCODE -ne 0) { throw 'Git submodule initialization failed.' } +Write-Host '[1/5] Preserving initialized WarcraftXL worktrees...' -New-Item -ItemType Directory -Path $PackagePath -Force | Out-Null +$previousPackageFiles = @( + Get-RelativeFilePaths -Root $PackagePath +) +if (-not $SkipDataBuild) { + Write-Host 'Resetting package staging to prevent stale build artifacts...' + Reset-MoonWellPackageStaging ` + -PackagePath $PackagePath ` + -RepositoryPath $repoRoot ` + -ClientPath $ClientPath +} else { + New-Item -ItemType Directory -Path $PackagePath -Force | Out-Null + Remove-MoonWellRuntimeGarbage -Destination $PackagePath +} if (-not $SkipDataBuild) { Write-Host '[2/5] Building MPQ patches...' @@ -99,6 +114,18 @@ if ($NativeRenderer) { $buildArguments.NativeRenderer = $true } if ($LASTEXITCODE -ne 0) { throw 'WarcraftXL deployment failed.' } Write-Host '[4/5] Synchronizing package files...' +$python = Get-Command python -ErrorAction SilentlyContinue +if (-not $python) { throw 'Python was not found; package validation cannot run.' } +& $python.Source (Join-Path $repoRoot 'validate_package.py') --dir $PackagePath +if ($LASTEXITCODE -ne 0) { throw 'Package validation failed.' } + +$currentPackageFiles = @(Get-RelativeFilePaths -Root $PackagePath) +Remove-StalePackageFilesFromClient ` + -ClientPath $ClientPath ` + -PreviousFiles $previousPackageFiles ` + -CurrentFiles $currentPackageFiles +Remove-MoonWellRuntimeGarbage -Destination $ClientPath + & robocopy $PackagePath $ClientPath /E /R:2 /W:1 /NFL /NDL /NJH /NJS /NP $robocopyExitCode = $LASTEXITCODE if ($robocopyExitCode -ge 8) { @@ -107,8 +134,6 @@ if ($robocopyExitCode -ge 8) { if (-not $SkipManifest) { Write-Host '[5/5] Building launcher manifest...' - $python = Get-Command python -ErrorAction SilentlyContinue - if (-not $python) { throw 'Python was not found; launcher manifest was not generated.' } & $python.Source (Join-Path $repoRoot 'build_manifest.py') ` --dir $PackagePath --output (Join-Path $repoRoot 'manifest.json') if ($LASTEXITCODE -ne 0) { throw 'Launcher manifest generation failed.' } diff --git a/patch-layout.json b/patch-layout.json new file mode 100644 index 0000000..bbc8f5f --- /dev/null +++ b/patch-layout.json @@ -0,0 +1,26 @@ +{ + "repositoryPatches": [ + "Data/patch-4.MPQ", + "Data/patch-Z.MPQ", + "Data/ruRU/patch-ruRU-4.MPQ", + "Data/ruRU/patch-ruRU-5.MPQ", + "Data/ruRU/patch-ruRU-Z.MPQ" + ], + "graphicsPatches": [ + "Data/patch-X.MPQ", + "Data/ruRU/backup-ruRU.MPQ", + "Data/ruRU/patch-ruRU-6.MPQ", + "Data/ruRU/patch-ruRU-A.mpq", + "Data/ruRU/patch-ruRU-B.mpq", + "Data/ruRU/patch-ruRU-E.mpq", + "Data/ruRU/patch-ruRU-H.MPQ", + "Data/ruRU/patch-ruRU-I.mpq", + "Data/ruRU/patch-ruRU-M.MPQ", + "Data/ruRU/patch-ruRU-S.mpq", + "Data/ruRU/patch-ruRU-T.mpq", + "Data/ruRU/patch-ruRU-U.mpq", + "Data/ruRU/patch-ruRU-W.mpq", + "Data/ruRU/patch-ruRU-X.MPQ", + "Data/ruRU/patch-ruRU-Y.MPQ" + ] +} diff --git a/patch_layout.py b/patch_layout.py new file mode 100644 index 0000000..27d8b50 --- /dev/null +++ b/patch_layout.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import json +from dataclasses import dataclass +from pathlib import Path + + +def normalized_path(path: str) -> str: + return path.replace("\\", "/").casefold() + + +@dataclass(frozen=True) +class PatchLayout: + repository_patches: frozenset[str] + graphics_patches: frozenset[str] + + def is_repository_patch(self, path: str) -> bool: + return normalized_path(path) in self.repository_patches + + def is_graphics_patch(self, path: str) -> bool: + return normalized_path(path) in self.graphics_patches + + def is_external_graphics_patch(self, path: str) -> bool: + key = normalized_path(path) + return key in self.graphics_patches and key not in self.repository_patches + + +def load_patch_layout(path: Path | None = None) -> PatchLayout: + config_path = path or Path(__file__).resolve().with_name("patch-layout.json") + data = json.loads(config_path.read_text(encoding="utf-8")) + repository = frozenset(normalized_path(item) for item in data["repositoryPatches"]) + graphics = frozenset(normalized_path(item) for item in data["graphicsPatches"]) + return PatchLayout(repository_patches=repository, graphics_patches=graphics) + + +PATCH_LAYOUT = load_patch_layout() diff --git a/pipeline-layout.ps1 b/pipeline-layout.ps1 new file mode 100644 index 0000000..4a6415d --- /dev/null +++ b/pipeline-layout.ps1 @@ -0,0 +1,241 @@ +$script:MoonWellExtensionNames = @( + 'MoonWell', + 'wxl-db2', + 'wxl-fdid-moonwell', + 'wxl-grasswind', + 'wxl-modern-adt', + 'wxl-modern-blp', + 'wxl-modern-m2', + 'wxl-modern-wmo', + 'wxl-moonwell-storage-fallback', + 'wxl-unit-outline' +) + +$patchLayoutPath = Join-Path $PSScriptRoot 'patch-layout.json' +if (-not (Test-Path -LiteralPath $patchLayoutPath -PathType Leaf)) { + throw "Patch layout was not found: $patchLayoutPath" +} +$patchLayout = Get-Content -LiteralPath $patchLayoutPath -Raw -Encoding UTF8 | ConvertFrom-Json +$script:MoonWellRepositoryPatches = @($patchLayout.repositoryPatches) +$script:MoonWellGraphicsPatches = @($patchLayout.graphicsPatches) + +function Get-MoonWellExtensionNames { + return @($script:MoonWellExtensionNames) +} + +function Get-MoonWellRepositoryPatchPaths { + return @($script:MoonWellRepositoryPatches) +} + +function Get-MoonWellGraphicsPatchPaths { + return @($script:MoonWellGraphicsPatches) +} + +function Test-MoonWellRepositoryPatch { + param([Parameter(Mandatory)][string]$RelativePath) + + foreach ($repositoryPatch in $script:MoonWellRepositoryPatches) { + if ($RelativePath.Replace('/', '\').Equals( + $repositoryPatch.Replace('/', '\'), + [System.StringComparison]::OrdinalIgnoreCase)) { + return $true + } + } + return $false +} + +function Test-MoonWellGraphicsPatch { + param([Parameter(Mandatory)][string]$RelativePath) + + foreach ($graphicsPatch in $script:MoonWellGraphicsPatches) { + if ($RelativePath.Replace('/', '\').Equals( + $graphicsPatch.Replace('/', '\'), + [System.StringComparison]::OrdinalIgnoreCase)) { + return $true + } + } + return $false +} + +function Test-MoonWellPreservedDataPatch { + param([Parameter(Mandatory)][string]$RelativePath) + + if (Test-MoonWellRepositoryPatch -RelativePath $RelativePath) { + return $false + } + return Test-MoonWellGraphicsPatch -RelativePath $RelativePath +} + +function Assert-SafePackagePath { + param( + [Parameter(Mandatory)] + [string]$PackagePath, + [Parameter(Mandatory)] + [string]$RepositoryPath, + [Parameter()] + [string]$ClientPath + ) + + $packageRoot = [System.IO.Path]::GetFullPath($PackagePath).TrimEnd('\') + $repositoryRoot = [System.IO.Path]::GetFullPath($RepositoryPath).TrimEnd('\') + $pathRoot = [System.IO.Path]::GetPathRoot($packageRoot).TrimEnd('\') + + if (-not $packageRoot -or $packageRoot -eq $pathRoot) { + throw "Refusing to use a filesystem root as package staging: $packageRoot" + } + if ($packageRoot.Equals($repositoryRoot, [System.StringComparison]::OrdinalIgnoreCase) -or + $repositoryRoot.StartsWith($packageRoot + '\', [System.StringComparison]::OrdinalIgnoreCase)) { + throw 'Package staging must not be the repository root or one of its ancestors.' + } + + if (-not [string]::IsNullOrWhiteSpace($ClientPath)) { + $clientRoot = [System.IO.Path]::GetFullPath($ClientPath).TrimEnd('\') + $packageIsClientOrChild = + $packageRoot.Equals($clientRoot, [System.StringComparison]::OrdinalIgnoreCase) -or + $packageRoot.StartsWith($clientRoot + '\', [System.StringComparison]::OrdinalIgnoreCase) + $packageContainsClient = + $clientRoot.StartsWith($packageRoot + '\', [System.StringComparison]::OrdinalIgnoreCase) + if ($packageIsClientOrChild -or $packageContainsClient) { + throw 'Package staging must be separate from the installed client tree.' + } + } +} + +function Get-RelativeFilePaths { + param([Parameter(Mandatory)][string]$Root) + + if (-not (Test-Path -LiteralPath $Root -PathType Container)) { + return @() + } + + $absoluteRoot = [System.IO.Path]::GetFullPath($Root).TrimEnd('\') + return @( + Get-ChildItem -LiteralPath $absoluteRoot -Recurse -Force -File | + ForEach-Object { $_.FullName.Substring($absoluteRoot.Length + 1) } + ) +} + +function Remove-EmptyMoonWellDirectories { + param([Parameter(Mandatory)][string]$Destination) + + foreach ($relativeRoot in @('Data\Patch-WXL.MPQ', 'Extensions', 'Utils')) { + $managedRoot = Join-Path $Destination $relativeRoot + if (-not (Test-Path -LiteralPath $managedRoot -PathType Container)) { + continue + } + + $directories = @( + Get-ChildItem -LiteralPath $managedRoot -Recurse -Force -Directory | + Sort-Object { $_.FullName.Length } -Descending + ) + $directories += Get-Item -LiteralPath $managedRoot + + foreach ($directory in $directories) { + if (-not @(Get-ChildItem -LiteralPath $directory.FullName -Force).Count) { + Remove-Item -LiteralPath $directory.FullName -Force + Write-Host "Removed empty managed directory: $($directory.FullName)" + } + } + } +} + +function Reset-MoonWellPackageStaging { + param( + [Parameter(Mandatory)][string]$PackagePath, + [Parameter(Mandatory)][string]$RepositoryPath, + [Parameter(Mandatory)][string]$ClientPath + ) + + Assert-SafePackagePath ` + -PackagePath $PackagePath ` + -RepositoryPath $RepositoryPath ` + -ClientPath $ClientPath + + if (Test-Path -LiteralPath $PackagePath) { + Remove-Item -LiteralPath $PackagePath -Recurse -Force + } + New-Item -ItemType Directory -Path $PackagePath -Force | Out-Null +} + +function Remove-MoonWellRuntimeGarbage { + param([Parameter(Mandatory)][string]$Destination) + + if (-not (Test-Path -LiteralPath $Destination -PathType Container)) { + return + } + + $allowedExtensions = @{} + foreach ($name in $script:MoonWellExtensionNames) { + $allowedExtensions[$name.ToLowerInvariant()] = $true + } + + $extensionsRoot = Join-Path $Destination 'Extensions' + if (Test-Path -LiteralPath $extensionsRoot -PathType Container) { + foreach ($entry in Get-ChildItem -LiteralPath $extensionsRoot -Force) { + $isAllowedDirectory = $entry.PSIsContainer -and + $allowedExtensions.ContainsKey($entry.Name.ToLowerInvariant()) + if (-not $isAllowedDirectory) { + Remove-Item -LiteralPath $entry.FullName -Recurse -Force + Write-Host "Removed unmanaged WarcraftXL extension artifact: $($entry.FullName)" + continue + } + + foreach ($dll in Get-ChildItem -LiteralPath $entry.FullName -Force -File -Filter '*.dll') { + $expectedName = "$($entry.Name).dll" + if (-not $dll.Name.Equals($expectedName, [System.StringComparison]::OrdinalIgnoreCase)) { + Remove-Item -LiteralPath $dll.FullName -Force + Write-Host "Removed stale extension DLL: $($dll.FullName)" + } + } + } + } + + foreach ($dll in Get-ChildItem -LiteralPath $Destination -Force -File -Filter 'WarcraftXL*.dll') { + if (-not $dll.Name.Equals('WarcraftXL.dll', [System.StringComparison]::OrdinalIgnoreCase)) { + Remove-Item -LiteralPath $dll.FullName -Force + Write-Host "Removed stale WarcraftXL DLL: $($dll.FullName)" + } + } + + $utilsRoot = Join-Path $Destination 'Utils' + if (Test-Path -LiteralPath $utilsRoot -PathType Container) { + foreach ($legacyFile in Get-ChildItem -LiteralPath $utilsRoot -Force -File -Filter 'WarcraftXLHost*') { + Remove-Item -LiteralPath $legacyFile.FullName -Force + Write-Host "Removed obsolete WarcraftXL 1.0 host artifact: $($legacyFile.FullName)" + } + } + + Remove-EmptyMoonWellDirectories -Destination $Destination +} + +function Remove-StalePackageFilesFromClient { + param( + [Parameter(Mandatory)][string]$ClientPath, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$PreviousFiles, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$CurrentFiles + ) + + $current = @{} + foreach ($relativePath in $CurrentFiles) { + $current[$relativePath.ToLowerInvariant()] = $true + } + + $clientRoot = [System.IO.Path]::GetFullPath($ClientPath).TrimEnd('\') + foreach ($relativePath in $PreviousFiles) { + if ($current.ContainsKey($relativePath.ToLowerInvariant()) -or + (Test-MoonWellPreservedDataPatch -RelativePath $relativePath)) { + continue + } + + $target = [System.IO.Path]::GetFullPath((Join-Path $clientRoot $relativePath)) + if (-not $target.StartsWith($clientRoot + '\', [System.StringComparison]::OrdinalIgnoreCase)) { + throw "Package inventory contains a path outside the client: $relativePath" + } + if (Test-Path -LiteralPath $target -PathType Leaf) { + Remove-Item -LiteralPath $target -Force + Write-Host "Removed stale packaged file from client: $target" + } + } + + Remove-EmptyMoonWellDirectories -Destination $ClientPath +} diff --git a/run.ps1 b/run.ps1 index f24551d..c720a11 100644 --- a/run.ps1 +++ b/run.ps1 @@ -9,6 +9,7 @@ $ErrorActionPreference = "Stop" # --- Resolve project root (folder where script is located) $ROOT = Split-Path -Parent $MyInvocation.MyCommand.Path +. (Join-Path $ROOT 'pipeline-layout.ps1') # --- Load .env if WOW_HOME not already set if (-not $env:WOW_HOME) { @@ -80,6 +81,11 @@ $TOOL = Join-Path $ROOT "tool\target\release\tool.exe" $RELOAD_SCRIPT = Join-Path $ROOT "reload_wow.bat" $WXL_BUILD_SCRIPT = Join-Path $ROOT "build-warcraftxl.ps1" +Assert-SafePackagePath ` + -PackagePath $DIST_DIR ` + -RepositoryPath $ROOT ` + -ClientPath $WOW_HOME + function Stop-WowRuntime { param([Parameter(Mandatory=$true)][string]$ClientPath) @@ -118,7 +124,14 @@ if (!(Test-Path $TOOL)) { Pop-Location } -# --- Build MPQ archives into dist/ +# --- Build MPQ archives into a clean dist/ +$previousPackageFiles = @( + Get-RelativeFilePaths -Root $DIST_DIR +) +Reset-MoonWellPackageStaging ` + -PackagePath $DIST_DIR ` + -RepositoryPath $ROOT ` + -ClientPath $WOW_HOME Write-Host "Building MPQ archives from src/ -> dist/..." & $TOOL $SRC_DIR $DIST_DIR if ($LASTEXITCODE -ne 0) { @@ -145,6 +158,22 @@ if ($LASTEXITCODE -ne 0) { # --- Sync dist/ -> WOW_HOME Write-Host "Syncing dist/ -> WOW_HOME..." +$python = Get-Command python -ErrorAction SilentlyContinue +if (-not $python) { + Write-Error "Python is required for package validation." + exit 1 +} +& $python.Source (Join-Path $ROOT 'validate_package.py') --dir $DIST_DIR +if ($LASTEXITCODE -ne 0) { + Write-Error "Package validation failed!" + exit 1 +} +$currentPackageFiles = @(Get-RelativeFilePaths -Root $DIST_DIR) +Remove-StalePackageFilesFromClient ` + -ClientPath $WOW_HOME ` + -PreviousFiles $previousPackageFiles ` + -CurrentFiles $currentPackageFiles +Remove-MoonWellRuntimeGarbage -Destination $WOW_HOME robocopy $DIST_DIR $WOW_HOME /E /NFL /NDL # robocopy exit codes 0-7 are success/warnings; 8+ are errors if ($LASTEXITCODE -ge 8) { diff --git a/tool/src/main.rs b/tool/src/main.rs index a763e43..cab3d84 100644 --- a/tool/src/main.rs +++ b/tool/src/main.rs @@ -119,35 +119,6 @@ fn stage_loose_assets( Ok(()) } -fn stage_customization_archives( - project_root: &Path, - output_dir: &Path, -) -> Result<(), Box> { - let source_dir = project_root.join("new customization mpqs"); - let locale_dir = output_dir.join("Data").join("ruRU"); - let archives = [ - ("patch-a-001.mpq", "patch-ruRU-X.MPQ"), - ("patch-b-002.mpq", "patch-ruRU-Y.MPQ"), - ]; - - for (source_name, target_name) in archives { - let source = source_dir.join(source_name); - if !source.is_file() { - println!( - "Customization archive not staged (missing): {}", - source.display() - ); - continue; - } - fs::create_dir_all(&locale_dir)?; - let target = locale_dir.join(target_name); - fs::copy(&source, &target)?; - println!("Staged customization archive: {}", target.display()); - } - - Ok(()) -} - fn main() -> Result<(), Box> { let args: Vec = env::args().collect(); @@ -214,8 +185,6 @@ fn main() -> Result<(), Box> { } } - stage_customization_archives(&project_root, &output_dir)?; - println!("\nAll MPQ archives built successfully."); Ok(()) } diff --git a/upload_to_s3.py b/upload_to_s3.py index d97d80f..da5ceed 100644 --- a/upload_to_s3.py +++ b/upload_to_s3.py @@ -11,9 +11,17 @@ import boto3 from botocore.config import Config from botocore.exceptions import ClientError +from patch_layout import PATCH_LAYOUT, normalized_path + S3_PREFIX = "World of Warcraft" MANIFEST_S3_KEY = "manifest.json" +MANAGED_MANIFEST_S3_KEY = "moonwell-managed-manifest.json" +LEGACY_MANAGED_FILES = { + "utils/warcraftxlhost.exe", + "utils/warcraftxlhost.log", + "warcraftxl.before-texture-upload-fix.dll", +} def compute_build_hash(files: list[dict]) -> str: lines = [ @@ -25,15 +33,59 @@ def compute_build_hash(files: list[dict]) -> str: def normalized_path_key(path: str) -> str: """Match client paths using Windows filesystem semantics.""" - return path.replace("\\", "/").casefold() + return normalized_path(path) -def merge_manifests(base_manifest: dict, staging_manifest: dict) -> dict: +def missing_repository_patches(manifest: dict) -> list[str]: + paths = { + normalized_path_key(item["path"]) + for item in manifest.get("files", []) + if isinstance(item, dict) and isinstance(item.get("path"), str) + } + return sorted(PATCH_LAYOUT.repository_patches - paths) + + +def external_graphics_patches(manifest: dict) -> list[str]: + return sorted( + item["path"] + for item in manifest.get("files", []) + if isinstance(item, dict) + and isinstance(item.get("path"), str) + and PATCH_LAYOUT.is_external_graphics_patch(item["path"]) + ) + + +def is_legacy_managed_path(path: str) -> bool: + key = normalized_path_key(path) + return ( + key in LEGACY_MANAGED_FILES + or key in {"wow.exe", "d3d9.dll", "warcraftxl.dll", "utils/d3d9-native.dll"} + or key.startswith("extensions/") + or key.startswith("data/patch-wxl.mpq/") + ) + + +def merge_manifests( + base_manifest: dict, + staging_manifest: dict, + previous_managed_paths: list[str] | None = None, +) -> dict: + previous_managed_keys = { + normalized_path_key(path) for path in (previous_managed_paths or []) + } files_by_path = { normalized_path_key(item["path"]): item for item in base_manifest.get("files", []) + if not ( + normalized_path_key(item["path"]) in previous_managed_keys + or is_legacy_managed_path(item["path"]) + or PATCH_LAYOUT.is_repository_patch(item["path"]) + ) + or PATCH_LAYOUT.is_external_graphics_patch(item["path"]) } for item in staging_manifest.get("files", []): + if PATCH_LAYOUT.is_external_graphics_patch(item["path"]): + continue files_by_path[normalized_path_key(item["path"])] = item files = sorted(files_by_path.values(), key=lambda item: item["path"].casefold()) @@ -43,6 +95,36 @@ def merge_manifests(base_manifest: dict, staging_manifest: dict) -> dict: } +def find_stale_managed_paths( + base_manifest: dict, + staging_manifest: dict, + previous_managed_paths: list[str] | None = None, +) -> list[str]: + previous_managed_keys = { + normalized_path_key(path) for path in (previous_managed_paths or []) + } + staging_keys = { + normalized_path_key(item["path"]) + for item in staging_manifest.get("files", []) + } + return sorted( + ( + item["path"] + for item in base_manifest.get("files", []) + if ( + ( + normalized_path_key(item["path"]) in previous_managed_keys + or is_legacy_managed_path(item["path"]) + or PATCH_LAYOUT.is_repository_patch(item["path"]) + ) + and not PATCH_LAYOUT.is_external_graphics_patch(item["path"]) + ) + and normalized_path_key(item["path"]) not in staging_keys + ), + key=str.casefold, + ) + + def load_dotenv(env_path: Path) -> dict[str, str]: values: dict[str, str] = {} if not env_path.exists(): @@ -103,6 +185,24 @@ def main() -> int: print(f"ERROR: unable to read local manifest.json: {error}", file=sys.stderr) return 1 + missing_repository = missing_repository_patches(staging_manifest) + if missing_repository: + print( + "ERROR: local manifest is missing repository patches: " + + ", ".join(missing_repository), + file=sys.stderr, + ) + return 1 + + unexpected_graphics = external_graphics_patches(staging_manifest) + if unexpected_graphics: + print( + "ERROR: managed manifest contains externally managed graphics patches: " + + ", ".join(unexpected_graphics), + file=sys.stderr, + ) + return 1 + try: response = s3.get_object(Bucket=bucket, Key=MANIFEST_S3_KEY) production_manifest = json.loads(response["Body"].read()) @@ -116,7 +216,32 @@ def main() -> int: print(f"ERROR: invalid production manifest: {error}", file=sys.stderr) return 1 - merged_manifest = merge_manifests(production_manifest, staging_manifest) + try: + response = s3.get_object(Bucket=bucket, Key=MANAGED_MANIFEST_S3_KEY) + previous_managed_manifest = json.loads(response["Body"].read()) + previous_managed_paths = [ + item["path"] for item in previous_managed_manifest.get("files", []) + ] + except ClientError as error: + error_code = error.response.get("Error", {}).get("Code") + if error_code not in {"NoSuchKey", "404"}: + print(f"ERROR: unable to read managed manifest: {error}", file=sys.stderr) + return 1 + previous_managed_paths = [] + except (UnicodeDecodeError, json.JSONDecodeError, KeyError) as error: + print(f"ERROR: invalid managed manifest: {error}", file=sys.stderr) + return 1 + + merged_manifest = merge_manifests( + production_manifest, + staging_manifest, + previous_managed_paths, + ) + stale_managed_paths = find_stale_managed_paths( + production_manifest, + staging_manifest, + previous_managed_paths, + ) print( "Manifest merge: " f"production={len(production_manifest.get('files', []))}, " @@ -139,6 +264,9 @@ def main() -> int: for local_path in files: rel_path = local_path.relative_to(dist_dir).as_posix() + if PATCH_LAYOUT.is_external_graphics_patch(rel_path): + print(f"Skipping externally managed graphics patch: {rel_path}") + continue s3_key = f"{S3_PREFIX}/{rel_path}" size_mb = local_path.stat().st_size / (1024 * 1024) print(f"Uploading {rel_path} ({size_mb:.1f} MB) -> s3://{bucket}/{s3_key}") @@ -154,27 +282,50 @@ def main() -> int: print("\nFinished with errors; production manifest was not changed.") return 1 - manifest_path.write_text( - json.dumps(merged_manifest, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - newline="\n", + json_content_type = "application/json" + no_cache = "no-cache, no-store, must-revalidate" + + print( + f"Uploading managed package inventory -> " + f"s3://{bucket}/{MANAGED_MANIFEST_S3_KEY}" ) + try: + s3.put_object( + Bucket=bucket, + Key=MANAGED_MANIFEST_S3_KEY, + Body=(json.dumps(staging_manifest, ensure_ascii=False, indent=2) + "\n").encode("utf-8"), + ContentType=json_content_type, + CacheControl=no_cache, + ) + print(" OK") + except Exception as e: + print(f" ERROR: {e}", file=sys.stderr) + return 1 print(f"Uploading manifest.json -> s3://{bucket}/{MANIFEST_S3_KEY}") try: - s3.upload_file( - str(manifest_path), - bucket, - MANIFEST_S3_KEY, - ExtraArgs={ - "ContentType": "application/json", - "CacheControl": "no-cache, no-store, must-revalidate", - }, + s3.put_object( + Bucket=bucket, + Key=MANIFEST_S3_KEY, + Body=(json.dumps(merged_manifest, ensure_ascii=False, indent=2) + "\n").encode("utf-8"), + ContentType=json_content_type, + CacheControl=no_cache, ) print(f" OK") except Exception as e: print(f" ERROR: {e}", file=sys.stderr) - errors = True + return 1 + + if stale_managed_paths: + print(f"Deleting {len(stale_managed_paths)} stale managed object(s)...") + for relative_path in stale_managed_paths: + s3_key = f"{S3_PREFIX}/{relative_path.replace(chr(92), '/')}" + try: + s3.delete_object(Bucket=bucket, Key=s3_key) + print(f" Deleted {s3_key}") + except Exception as e: + print(f" ERROR deleting {s3_key}: {e}", file=sys.stderr) + errors = True if errors: print("\nFinished with errors.") diff --git a/validate_package.py b/validate_package.py new file mode 100644 index 0000000..7598116 --- /dev/null +++ b/validate_package.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +from patch_layout import PATCH_LAYOUT, normalized_path + + +EXTENSIONS = { + "MoonWell", + "wxl-db2", + "wxl-fdid-moonwell", + "wxl-grasswind", + "wxl-modern-adt", + "wxl-modern-blp", + "wxl-modern-m2", + "wxl-modern-wmo", + "wxl-moonwell-storage-fallback", + "wxl-unit-outline", +} +EXTENSION_KEYS = {name.casefold() for name in EXTENSIONS} + +ALLOWED_TOP_LEVEL_FILES = { + "d3d9.dll", + "warcraftxl.dll", + "wow.exe", +} +ALLOWED_TOP_LEVEL_DIRS = {"data", "extensions", "utils"} + +REQUIRED_RUNTIME_FILES = { + "wow.exe", + "d3d9.dll", + "warcraftxl.dll", + *PATCH_LAYOUT.repository_patches, + "utils/d3d9-native.dll", + *(f"extensions/{name.casefold()}/{name.casefold()}.dll" for name in EXTENSIONS), +} + +ALLOWED_DLL_FILES = { + path for path in REQUIRED_RUNTIME_FILES if path.endswith(".dll") +} + +BUILD_ONLY_SUFFIXES = { + ".bak", + ".exp", + ".ilk", + ".lib", + ".log", + ".obj", + ".old", + ".orig", + ".pdb", + ".pyc", + ".rej", + ".temp", + ".tmp", +} + + +def normalized(path: Path) -> str: + return normalized_path(path.as_posix()) + + +def validate(root: Path) -> list[str]: + errors: list[str] = [] + files = sorted(path for path in root.rglob("*") if path.is_file()) + relative_files = {normalized(path.relative_to(root)) for path in files} + + for missing in sorted(REQUIRED_RUNTIME_FILES - relative_files): + errors.append(f"missing required runtime file: {missing}") + + for entry in root.iterdir(): + key = entry.name.casefold() + if entry.is_file() and key not in ALLOWED_TOP_LEVEL_FILES: + errors.append(f"unexpected top-level file: {entry.name}") + elif entry.is_dir() and key not in ALLOWED_TOP_LEVEL_DIRS: + errors.append(f"unexpected top-level directory: {entry.name}") + + for path in files: + relative = path.relative_to(root) + relative_key = normalized(relative) + parts = relative.parts + + if PATCH_LAYOUT.is_external_graphics_patch(relative.as_posix()): + errors.append( + f"externally managed graphics patch in repository package: {relative.as_posix()}" + ) + + if path.suffix.casefold() in BUILD_ONLY_SUFFIXES: + errors.append(f"build-only or temporary file: {relative.as_posix()}") + + if path.suffix.casefold() == ".dll" and relative_key not in ALLOWED_DLL_FILES: + errors.append(f"unexpected DLL: {relative.as_posix()}") + + if path.suffix.casefold() == ".exe" and relative_key != "wow.exe": + errors.append(f"unexpected executable: {relative.as_posix()}") + + if parts and parts[0].casefold() == "extensions": + if len(parts) != 3: + errors.append(f"unexpected Extensions layout: {relative.as_posix()}") + continue + extension_name = parts[1] + expected_file = f"{extension_name}.dll" + if extension_name.casefold() not in EXTENSION_KEYS: + errors.append(f"unexpected extension: {relative.as_posix()}") + elif parts[2].casefold() != expected_file.casefold(): + errors.append(f"unexpected file in extension package: {relative.as_posix()}") + + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description="Validate MoonWell package layout") + parser.add_argument("--dir", required=True, help="Package staging directory") + args = parser.parse_args() + + root = Path(args.dir).resolve() + if not root.is_dir(): + print(f"ERROR: package directory not found: {root}", file=sys.stderr) + return 1 + + errors = validate(root) + if errors: + print("ERROR: package validation failed:", file=sys.stderr) + for error in errors: + print(f" - {error}", file=sys.stderr) + return 1 + + print(f"OK: package layout validated ({len(list(root.rglob('*')))} entries)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/modules/wxl-db2 b/vendor/modules/wxl-db2 index b3b4e31..30e4f2c 160000 --- a/vendor/modules/wxl-db2 +++ b/vendor/modules/wxl-db2 @@ -1 +1 @@ -Subproject commit b3b4e31cc43218faf47745facc4d7e4d65dfbbf1 +Subproject commit 30e4f2c8ed887a8d8d4b4472d688d4b8a02c310d diff --git a/vendor/modules/wxl-modern-m2 b/vendor/modules/wxl-modern-m2 index ca841be..b5a823a 160000 --- a/vendor/modules/wxl-modern-m2 +++ b/vendor/modules/wxl-modern-m2 @@ -1 +1 @@ -Subproject commit ca841be8a928b2f717b088c0b5efd670daf640df +Subproject commit b5a823a44b6e15208ba75b92a79845db9a5308e7 diff --git a/vendor/warcraftxl b/vendor/warcraftxl index 4895cef..1508beb 160000 --- a/vendor/warcraftxl +++ b/vendor/warcraftxl @@ -1 +1 @@ -Subproject commit 4895cef6f41ade7fc89946d21009fed9f4fcfbe0 +Subproject commit 1508beb6ec1eeb337a243f22fe6fe27deb9a8cf0