4 Commits

Author SHA1 Message Date
sindoring 597f4030a7 dvc 2026-08-22 19:10:11 +04:00
gasaichandesu ebd76275a1 fix(transmog): remove embed.xml from Transmogrification.toc because this
file does not exist anymore
2026-08-19 20:28:53 +04:00
gasaichandesu 6addd9bb51 feat(premium): add summon squire spell to Spell.dbc 2026-08-19 16:59:32 +04:00
sindoring a984e705c3 обновление сборщика 2026-08-16 16:56:34 +04:00
49 changed files with 1686 additions and 89 deletions
+3
View File
@@ -0,0 +1,3 @@
/config.local
/tmp
/cache
+8
View File
@@ -0,0 +1,8 @@
[cache]
type = "hardlink,copy"
[core]
remote = customization
['remote "customization"']
url = s3://warcraft-client/dvc/moonwell-client-sources/customization
endpointurl = https://storage.yandexcloud.net
region = ru-central1
+3
View File
@@ -0,0 +1,3 @@
# Add patterns of files dvc should ignore, which could improve
# the performance. Learn more at
# https://dvc.org/doc/user-guide/dvcignore
+4
View File
@@ -2,6 +2,7 @@ WOW_HOME=C:\Games\World of Warcraft 3.3.5
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_SESSION_TOKEN=
AWS_DEFAULT_REGION=
AWS_BUCKET=
AWS_ENDPOINT=
@@ -10,3 +11,6 @@ AWS_USE_PATH_STYLE_ENDPOINT=
PRODUCTION_REALMLIST=
PTR_REALMLIST=
LOCAL_REALMLIST=
SSH_USER=root
SSH_HOST=strictlypragmatic.ru
+1
View File
@@ -5,6 +5,7 @@ manifest.json
.vscode
dist
build/
/new customization mpqs/*.mpq
Wow*.exe
*.backup.exe
Logs/
+2 -1
View File
@@ -15,7 +15,8 @@
url = https://github.com/WarcraftXL/wxl-db2.git
[submodule "vendor/modules/wxl-modern-m2"]
path = vendor/modules/wxl-modern-m2
url = https://github.com/WarcraftXL/wxl-modern-m2.git
url = https://github.com/sindoring/wxl-modern-m2.git
branch = agent/guard-oversized-m2-batches
[submodule "vendor/modules/wxl-modern-wmo"]
path = vendor/modules/wxl-modern-wmo
url = https://github.com/WarcraftXL/wxl-modern-wmo.git
+9 -8
View File
@@ -283,8 +283,7 @@ build/customization-compat/server-dbc/BarberShopStyle.dbc
Упаковщик:
- собирает Z из исходников и сгенерированных MoonWell DBC;
- копирует `patch-a-001.mpq` в `dist\Data\ruRU\patch-ruRU-X.MPQ`;
- копирует `patch-b-002.mpq` в `dist\Data\ruRU\patch-ruRU-Y.MPQ`;
- не копирует внешние графические X/Y в `dist` и не включает их в managed manifest;
- не добавляет `patch-k.mpq`.
Для полной локальной сборки и синхронизации клиента можно использовать:
@@ -303,13 +302,15 @@ build/customization-compat/server-dbc/BarberShopStyle.dbc
## Исходники MPQ и установленные копии
Файлы в `new customization mpqs` — источники сборки. В готовом клиенте
используются их копии X и Y.
Файлы в `new customization mpqs` используются только как локальные источники
генератора таблиц совместимости. В готовом клиенте X и Y управляются отдельно от
репозиторных патчей.
Технически уже установленный клиент может запускаться без исходников в
репозитории, если X/Y остаются в `Data\ruRU`. Однако удаление исходников сломает
повторную сборку `dist`, восстановление клиента и генерацию совместимости.
Поэтому `patch-a-001.mpq` и `patch-b-002.mpq` нужно хранить.
Списки репозиторных и внешних графических патчей находятся в
`patch-layout.json`. `deploy.ps1`, `run.ps1`, генератор манифеста и S3 uploader
синхронизируют только список `repositoryPatches`. Пути из `graphicsPatches`
сохраняются без копирования и удаления. Если один путь указан в обоих списках,
`repositoryPatches` имеет приоритет.
Z не консолидирует многогигабайтные модели и текстуры X/Y. Удаление X или Y из
клиента не освобождается наличием текущих DBC в Z.
+12
View File
@@ -47,6 +47,8 @@ git submodule update --remote vendor/warcraftxl vendor/modules/*
- Visual Studio 2022 с C++ toolchain для Win32;
- CMake 3.20+;
- Rust/Cargo для существующего MPQ-сборщика;
- Python 3 и DVC с поддержкой S3 для содержимого внешних графических патчей
(`py -m pip install -r requirements-dvc.txt`);
- локальный оригинальный `Wow_Original.exe` с SHA-256
`AA63A5750D60EF16746C686B3D5E26876D98953EAB08B1C026CD0FAF78E88CB8`.
@@ -123,6 +125,16 @@ WarcraftXL не требуется и в объектное хранилище
- `patch-ruRU-5` — русская локализация и интерфейс;
- `patch-Z` — Mythic+ ресурсы и клиентские данные.
Управляемые репозиторием MPQ и внешние графические патчи перечислены отдельно в
`patch-layout.json`. Сборка и deploy синхронизируют только `repositoryPatches` и
не изменяют `graphicsPatches`. При пересечении списков репозиторный патч имеет
приоритет.
Распакованное содержимое больших патчей кастомизации хранится через DVC в
`assets/graphics/`, а бинарные объекты — в настроенном S3. Инструкции по первичной
загрузке, изменению и импорту MPQ находятся в
[`assets/graphics/README.md`](assets/graphics/README.md).
## Лицензирование
WarcraftXL распространяется по GPL-3.0 и подключён как отдельный неизменённый upstream submodule.
+1
View File
@@ -0,0 +1 @@
/patch-X
+6
View File
@@ -0,0 +1,6 @@
outs:
- md5: 012ce72d8dfdf089b20a28f6e6158b58.dir
size: 1195310
nfiles: 4
hash: md5
path: patch-X
+14
View File
@@ -0,0 +1,14 @@
/patch-ruRU-X
/patch-ruRU-Y
/backup-ruRU
/patch-ruRU-6
/patch-ruRU-A
/patch-ruRU-B
/patch-ruRU-E
/patch-ruRU-H
/patch-ruRU-I
/patch-ruRU-M
/patch-ruRU-S
/patch-ruRU-T
/patch-ruRU-U
/patch-ruRU-W
@@ -0,0 +1,6 @@
outs:
- md5: b120587dfcd255f78392ca52705a00ff.dir
size: 38876467
nfiles: 161
hash: md5
path: backup-ruRU
@@ -0,0 +1,6 @@
outs:
- md5: 9ca2ee746d80e8a0c951c5ef476ae654.dir
size: 1064210
nfiles: 3
hash: md5
path: patch-ruRU-6
@@ -0,0 +1,6 @@
outs:
- md5: 97232ae859eaa1a16b137efd1a2e9122.dir
size: 2748988973
nfiles: 8817
hash: md5
path: patch-ruRU-A
@@ -0,0 +1,6 @@
outs:
- md5: bb686aebf7c9cae2c02433abde9a9f19.dir
size: 771051083
nfiles: 2911
hash: md5
path: patch-ruRU-B
@@ -0,0 +1,6 @@
outs:
- md5: d6b2fe0e3f23c47dc944ca4a8f82d97a.dir
size: 9275755440
nfiles: 21977
hash: md5
path: patch-ruRU-E
@@ -0,0 +1,6 @@
outs:
- md5: 5e95536d2c6f6ff792f05b800105bedc.dir
size: 3727395837
nfiles: 40296
hash: md5
path: patch-ruRU-H
@@ -0,0 +1,6 @@
outs:
- md5: 5db07768b0814c970d84a99b6072fd9f.dir
size: 31453961
nfiles: 186
hash: md5
path: patch-ruRU-I
@@ -0,0 +1,6 @@
outs:
- md5: e927ae98c9971f378a82feb3299fdcb8.dir
size: 69521651
nfiles: 1552
hash: md5
path: patch-ruRU-M
@@ -0,0 +1,6 @@
outs:
- md5: 449383bbe05ebebb6bad2c37da7793c4.dir
size: 142167464
nfiles: 1751
hash: md5
path: patch-ruRU-S
@@ -0,0 +1,6 @@
outs:
- md5: fc19500b5ce0fd312a415a3118afd58d.dir
size: 656858260
nfiles: 1810
hash: md5
path: patch-ruRU-T
@@ -0,0 +1,6 @@
outs:
- md5: 783909a6c2ebed5d4f43516fe1ea68bb.dir
size: 460642379
nfiles: 1837
hash: md5
path: patch-ruRU-U
@@ -0,0 +1,6 @@
outs:
- md5: e2a1448eb0e443797acd577fc4b67249.dir
size: 44597910
nfiles: 484
hash: md5
path: patch-ruRU-W
@@ -0,0 +1,6 @@
outs:
- md5: 3d322d290959d1ead612f3b76860a1ea.dir
size: 3984134962
nfiles: 29495
hash: md5
path: patch-ruRU-X
@@ -0,0 +1,6 @@
outs:
- md5: 42f3a8ac35674e33c0a4420c84cadcb8.dir
size: 11053212781
nfiles: 36191
hash: md5
path: patch-ruRU-Y
+66
View File
@@ -0,0 +1,66 @@
# Графические MPQ-патчи
В этой директории через DVC версионируется распакованное содержимое всех 15 патчей из
`graphicsPatches` в `patch-layout.json`. Git хранит небольшие файлы `*.dvc`, а сами ресурсы —
в S3 remote `customization`. Хэшированные пути в S3 являются внутренней адресацией DVC:
исходное дерево с обычными именами восстанавливается командой `dvc pull`.
## Первичная настройка
```powershell
py -m pip install -r requirements-dvc.txt
Copy-Item .env.example .env # если локального .env ещё нет
.\configure-dvc.ps1
py -m dvc pull
```
Можно скачать один патч, передав его указатель явно:
```powershell
py -m dvc pull assets/graphics/Data/ruRU/patch-ruRU-X.dvc
```
Секреты записываются только в `.dvc/config.local`; этот файл исключён из Git. Вместо ключей
из `.env` можно использовать AWS profile: `.\configure-dvc.ps1 -Profile moonwell`.
## Изменение ресурсов
DVC-кэш использует hardlink на NTFS. Перед ручным изменением дерева отвяжите его от кэша:
```powershell
py -m dvc unprotect assets/graphics/Data/ruRU/patch-ruRU-X
# изменить, добавить или удалить файлы
py -m dvc add assets/graphics/Data/ruRU/patch-ruRU-X
py -m dvc push
git add assets/graphics/Data/ruRU/patch-ruRU-X.dvc
```
Сначала выполняйте `dvc push`, затем коммитьте Git-указатель, чтобы коммит не ссылался на ещё
не загруженные объекты.
## Импорт MPQ
`mpqextract` принимает встроенный `(listfile)` и любое количество внешних listfile. Он извлекает
все известные пути, а блоки с неизвестными именами сохраняет в `__mpqmeta__/unresolved/`.
Манифест `__mpqmeta__/unresolved.json` связывает каждый blob с MPQ-хэшами и индексом блока,
поэтому содержимое архива не теряется даже при неполном listfile.
```powershell
cargo run --release --manifest-path tool/Cargo.toml --bin mpqextract -- `
'patch.mpq' 'assets/graphics/Data/ruRU/patch-ruRU-N' 'master-list.txt'
```
В `patch-ruRU-A`, `B`, `S`, `T`, `U` и `W` есть такие безымянные блоки. Для их безопасной
пересборки нужен авторитетный listfile с исходными путями; пока он не найден, исходные MPQ в
дистрибуционном S3 остаются каноническими архивами. `mpqpack` намеренно отказывается собирать
дерево с `__mpqmeta__`, поскольку подмена путей хэшами изменила бы семантику патча.
Полностью разрешённое дерево собирается так:
```powershell
cargo run --release --manifest-path tool/Cargo.toml --bin mpqpack -- `
'assets/graphics/Data/ruRU/patch-ruRU-X' 'build/graphics/patch-ruRU-X.MPQ'
```
Сводка исходных объектов, DVC-хэшей и количества разрешённых блоков находится в
`assets/graphics/sources.json`.
+157
View File
@@ -0,0 +1,157 @@
{
"sourcePrefix": "s3://warcraft-client/World of Warcraft/",
"patches": [
{
"archive": "Data/patch-X.MPQ",
"archiveBytes": 496584,
"dvcPointer": "assets/graphics/Data/patch-X.dvc",
"dvcMd5": "012ce72d8dfdf089b20a28f6e6158b58.dir",
"extractedBytes": 1195310,
"trackedFiles": 4,
"resolvedFiles": 4,
"unresolvedBlocks": 0
},
{
"archive": "Data/ruRU/backup-ruRU.MPQ",
"archiveBytes": 23981093,
"dvcPointer": "assets/graphics/Data/ruRU/backup-ruRU.dvc",
"dvcMd5": "b120587dfcd255f78392ca52705a00ff.dir",
"extractedBytes": 38876467,
"trackedFiles": 161,
"resolvedFiles": 161,
"unresolvedBlocks": 0
},
{
"archive": "Data/ruRU/patch-ruRU-6.MPQ",
"archiveBytes": 375062,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-6.dvc",
"dvcMd5": "9ca2ee746d80e8a0c951c5ef476ae654.dir",
"extractedBytes": 1064210,
"trackedFiles": 3,
"resolvedFiles": 3,
"unresolvedBlocks": 0
},
{
"archive": "Data/ruRU/patch-ruRU-A.mpq",
"archiveBytes": 1451338841,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-A.dvc",
"dvcMd5": "97232ae859eaa1a16b137efd1a2e9122.dir",
"extractedBytes": 2748988973,
"trackedFiles": 8817,
"resolvedFiles": 5649,
"unresolvedBlocks": 3167
},
{
"archive": "Data/ruRU/patch-ruRU-B.mpq",
"archiveBytes": 384938550,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-B.dvc",
"dvcMd5": "bb686aebf7c9cae2c02433abde9a9f19.dir",
"extractedBytes": 771051083,
"trackedFiles": 2911,
"resolvedFiles": 1679,
"unresolvedBlocks": 1231
},
{
"archive": "Data/ruRU/patch-ruRU-E.mpq",
"archiveBytes": 1862937450,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-E.dvc",
"dvcMd5": "d6b2fe0e3f23c47dc944ca4a8f82d97a.dir",
"extractedBytes": 9275755440,
"trackedFiles": 21977,
"resolvedFiles": 21977,
"unresolvedBlocks": 0
},
{
"archive": "Data/ruRU/patch-ruRU-H.MPQ",
"archiveBytes": 2154531136,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-H.dvc",
"dvcMd5": "5e95536d2c6f6ff792f05b800105bedc.dir",
"extractedBytes": 3727395837,
"trackedFiles": 40296,
"resolvedFiles": 40296,
"unresolvedBlocks": 0
},
{
"archive": "Data/ruRU/patch-ruRU-I.mpq",
"archiveBytes": 14891516,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-I.dvc",
"dvcMd5": "5db07768b0814c970d84a99b6072fd9f.dir",
"extractedBytes": 31453961,
"trackedFiles": 186,
"resolvedFiles": 186,
"unresolvedBlocks": 0
},
{
"archive": "Data/ruRU/patch-ruRU-M.MPQ",
"archiveBytes": 48772900,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-M.dvc",
"dvcMd5": "e927ae98c9971f378a82feb3299fdcb8.dir",
"extractedBytes": 69521651,
"trackedFiles": 1552,
"resolvedFiles": 1552,
"unresolvedBlocks": 0
},
{
"archive": "Data/ruRU/patch-ruRU-S.mpq",
"archiveBytes": 80492633,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-S.dvc",
"dvcMd5": "449383bbe05ebebb6bad2c37da7793c4.dir",
"extractedBytes": 142167464,
"trackedFiles": 1751,
"resolvedFiles": 1432,
"unresolvedBlocks": 318
},
{
"archive": "Data/ruRU/patch-ruRU-T.mpq",
"archiveBytes": 418595462,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-T.dvc",
"dvcMd5": "fc19500b5ce0fd312a415a3118afd58d.dir",
"extractedBytes": 656858260,
"trackedFiles": 1810,
"resolvedFiles": 1731,
"unresolvedBlocks": 78
},
{
"archive": "Data/ruRU/patch-ruRU-U.mpq",
"archiveBytes": 233678341,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-U.dvc",
"dvcMd5": "783909a6c2ebed5d4f43516fe1ea68bb.dir",
"extractedBytes": 460642379,
"trackedFiles": 1837,
"resolvedFiles": 1372,
"unresolvedBlocks": 464
},
{
"archive": "Data/ruRU/patch-ruRU-W.mpq",
"archiveBytes": 29433979,
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-W.dvc",
"dvcMd5": "e2a1448eb0e443797acd577fc4b67249.dir",
"extractedBytes": 44597910,
"trackedFiles": 484,
"resolvedFiles": 381,
"unresolvedBlocks": 102
},
{
"archive": "Data/ruRU/patch-ruRU-X.MPQ",
"archiveBytes": 2673854694,
"archiveSha256": "CD8CED6FA6C863D509DAE458E6DD44C3986B8B57731B6DC6192FE94125F8C1FD",
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-X.dvc",
"dvcMd5": "3d322d290959d1ead612f3b76860a1ea.dir",
"extractedBytes": 3984134962,
"trackedFiles": 29495,
"resolvedFiles": 29495,
"unresolvedBlocks": 0
},
{
"archive": "Data/ruRU/patch-ruRU-Y.MPQ",
"archiveBytes": 3001665276,
"archiveSha256": "534D496C1A1EE023B910ABC80B727BA88BB09CD2F5767100FC18EE41F4306B90",
"dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-Y.dvc",
"dvcMd5": "42f3a8ac35674e33c0a4420c84cadcb8.dir",
"extractedBytes": 11053212781,
"trackedFiles": 36191,
"resolvedFiles": 36191,
"unresolvedBlocks": 0
}
]
}
+53 -20
View File
@@ -75,8 +75,53 @@ function Enable-LargeAddressAware {
}
}
function Initialize-MissingSubmodules {
param(
[Parameter(Mandatory)]
[string]$RepositoryPath
)
$modulesFile = Join-Path $RepositoryPath '.gitmodules'
if (-not (Test-Path -LiteralPath $modulesFile -PathType Leaf)) {
return
}
$configuredModules = @(
& git -C $RepositoryPath config --file .gitmodules --get-regexp '^submodule\..*\.path$'
)
if ($LASTEXITCODE -gt 1) {
throw "Failed to read submodules from $modulesFile"
}
foreach ($configuredModule in $configuredModules) {
$fields = $configuredModule -split '\s+', 2
if ($fields.Count -ne 2) {
continue
}
$relativePath = $fields[1].Trim()
$modulePath = Join-Path $RepositoryPath $relativePath
$gitMarker = Join-Path $modulePath '.git'
# Updating an initialized submodule checks out the SHA recorded by its parent and detaches
# any active development branch. Only initialize genuinely missing worktrees.
if (-not (Test-Path -LiteralPath $gitMarker)) {
Write-Host "Initializing missing submodule: $modulePath"
& git -C $RepositoryPath submodule update --init -- $relativePath
if ($LASTEXITCODE -ne 0) {
throw "Failed to initialize submodule: $modulePath"
}
}
if (Test-Path -LiteralPath $modulePath -PathType Container) {
Initialize-MissingSubmodules -RepositoryPath $modulePath
}
}
}
$ErrorActionPreference = 'Stop'
$repoRoot = $PSScriptRoot
. (Join-Path $repoRoot 'pipeline-layout.ps1')
$cmake = 'C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe'
if (-not (Test-Path -LiteralPath $cmake)) {
$cmakeCommand = Get-Command cmake -ErrorAction SilentlyContinue
@@ -84,8 +129,7 @@ if (-not (Test-Path -LiteralPath $cmake)) {
$cmake = $cmakeCommand.Source
}
& git -C $repoRoot submodule update --init --recursive
if ($LASTEXITCODE -ne 0) { throw 'Failed to initialize WarcraftXL submodules.' }
Initialize-MissingSubmodules -RepositoryPath $repoRoot
$win32BuildDir = Join-Path $repoRoot 'build\warcraftxl-win32'
$stockExe = Join-Path $repoRoot 'Wow_Original.exe'
@@ -110,18 +154,7 @@ $win32ArtifactDir = Join-Path $win32BuildDir "vendor\warcraftxl\$Configuration"
$warcraftXL = Join-Path $win32ArtifactDir 'WarcraftXL.dll'
$selectedProxy = Join-Path $win32ArtifactDir 'd3d9.dll'
$recoveryProxy = Join-Path $win32BuildDir "artifacts\$Configuration\d3d9-native.dll"
$extensionNames = @(
'MoonWell',
'wxl-db2',
'wxl-fdid-moonwell',
'wxl-grasswind',
'wxl-modern-adt',
'wxl-modern-blp',
'wxl-modern-m2',
'wxl-modern-wmo',
'wxl-moonwell-storage-fallback',
'wxl-unit-outline'
)
$extensionNames = Get-MoonWellExtensionNames
if ($NativeRenderer) {
Write-Warning '-NativeRenderer is no longer needed: WarcraftXL 1.1 uses native D3D9 by default.'
@@ -133,6 +166,8 @@ function Install-WarcraftXLArtifacts {
[string]$Destination
)
Remove-MoonWellRuntimeGarbage -Destination $Destination
$utils = Join-Path $Destination 'Utils'
New-Item `
-ItemType Directory `
@@ -140,12 +175,6 @@ function Install-WarcraftXLArtifacts {
-Force |
Out-Null
$legacyHost = Join-Path $utils 'WarcraftXLHost.exe'
if (Test-Path -LiteralPath $legacyHost -PathType Leaf) {
Remove-Item -LiteralPath $legacyHost -Force
Write-Host "Removed obsolete WarcraftXL 1.0 host: $legacyHost"
}
$destinationExe = Join-Path $Destination 'Wow.exe'
Copy-Item `
@@ -208,6 +237,10 @@ if ($Deploy) {
if (-not [string]::IsNullOrWhiteSpace($PackagePath)) {
$PackagePath = [System.IO.Path]::GetFullPath($PackagePath)
Assert-SafePackagePath `
-PackagePath $PackagePath `
-RepositoryPath $repoRoot `
-ClientPath $ClientPath
Install-WarcraftXLArtifacts -Destination $PackagePath
Write-Host "Packaged stock Wow.exe and WarcraftXL artifacts in $PackagePath"
}
+4
View File
@@ -9,6 +9,8 @@ import sys
from pathlib import Path
from typing import Optional
from patch_layout import PATCH_LAYOUT
IGNORED_TOP_LEVEL_DIRS = {
"Cache",
@@ -49,6 +51,8 @@ def is_ignored(rel_path: Path) -> bool:
return True
if len(rel_path.parts) == 1 and rel_path.name in IGNORED_TOP_LEVEL_FILES:
return True
if PATCH_LAYOUT.is_external_graphics_patch(rel_path.as_posix()):
return True
return any(part in IGNORED_DIRS_ANYWHERE for part in rel_path.parts[:-1])
+56
View File
@@ -0,0 +1,56 @@
param(
[string]$EnvFile = (Join-Path $PSScriptRoot '.env'),
[string]$Profile
)
$ErrorActionPreference = 'Stop'
function Get-EnvValue([string]$Path, [string]$Name) {
if (-not (Test-Path -LiteralPath $Path)) {
return $null
}
$line = Get-Content -LiteralPath $Path |
Where-Object { $_ -match "^$([regex]::Escape($Name))=" } |
Select-Object -First 1
if ($line) {
return $line.Substring($Name.Length + 1).Trim()
}
return $null
}
function Invoke-Dvc([string[]]$Arguments) {
& py -m dvc @Arguments
if ($LASTEXITCODE -ne 0) {
throw "DVC command failed: dvc $($Arguments -join ' ')"
}
}
if (-not (Test-Path -LiteralPath (Join-Path $PSScriptRoot '.dvc\config'))) {
throw 'DVC is not initialized in this repository.'
}
& py -c 'import dvc, dvc_s3' *> $null
if ($LASTEXITCODE -ne 0) {
throw 'DVC with S3 support is not installed. Run: py -m pip install -r requirements-dvc.txt'
}
if ($Profile) {
Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'profile', $Profile)
} else {
$accessKey = Get-EnvValue $EnvFile 'AWS_ACCESS_KEY_ID'
$secretKey = Get-EnvValue $EnvFile 'AWS_SECRET_ACCESS_KEY'
$sessionToken = Get-EnvValue $EnvFile 'AWS_SESSION_TOKEN'
if (-not $accessKey -or -not $secretKey) {
throw "AWS credentials are missing in $EnvFile. Pass -Profile or fill AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY."
}
Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'access_key_id', $accessKey)
Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'secret_access_key', $secretKey)
if ($sessionToken) {
Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'session_token', $sessionToken)
}
}
Write-Host 'Local DVC credentials configured. They are stored in ignored .dvc/config.local.'
+31 -6
View File
@@ -13,6 +13,7 @@ param(
$ErrorActionPreference = 'Stop'
$repoRoot = $PSScriptRoot
. (Join-Path $repoRoot 'pipeline-layout.ps1')
function Import-ProjectEnvironment {
$envFile = Join-Path $repoRoot '.env'
@@ -46,6 +47,10 @@ $PackagePath = [System.IO.Path]::GetFullPath($PackagePath)
if (-not (Test-Path -LiteralPath $ClientPath -PathType Container)) {
throw "Client directory was not found: $ClientPath"
}
Assert-SafePackagePath `
-PackagePath $PackagePath `
-RepositoryPath $repoRoot `
-ClientPath $ClientPath
$runningClients = @(Get-Process Wow -ErrorAction SilentlyContinue | Where-Object {
$_.Path -and $_.Path.StartsWith($ClientPath, [System.StringComparison]::OrdinalIgnoreCase)
@@ -64,11 +69,21 @@ $env:WOW_HOME = $ClientPath
Write-Host "MoonWell client: $ClientPath"
Write-Host "Package staging: $PackagePath"
Write-Host '[1/5] Initializing WarcraftXL dependencies...'
& git -C $repoRoot submodule update --init --recursive
if ($LASTEXITCODE -ne 0) { throw 'Git submodule initialization failed.' }
Write-Host '[1/5] Preserving initialized WarcraftXL worktrees...'
New-Item -ItemType Directory -Path $PackagePath -Force | Out-Null
$previousPackageFiles = @(
Get-RelativeFilePaths -Root $PackagePath
)
if (-not $SkipDataBuild) {
Write-Host 'Resetting package staging to prevent stale build artifacts...'
Reset-MoonWellPackageStaging `
-PackagePath $PackagePath `
-RepositoryPath $repoRoot `
-ClientPath $ClientPath
} else {
New-Item -ItemType Directory -Path $PackagePath -Force | Out-Null
Remove-MoonWellRuntimeGarbage -Destination $PackagePath
}
if (-not $SkipDataBuild) {
Write-Host '[2/5] Building MPQ patches...'
@@ -99,6 +114,18 @@ if ($NativeRenderer) { $buildArguments.NativeRenderer = $true }
if ($LASTEXITCODE -ne 0) { throw 'WarcraftXL deployment failed.' }
Write-Host '[4/5] Synchronizing package files...'
$python = Get-Command python -ErrorAction SilentlyContinue
if (-not $python) { throw 'Python was not found; package validation cannot run.' }
& $python.Source (Join-Path $repoRoot 'validate_package.py') --dir $PackagePath
if ($LASTEXITCODE -ne 0) { throw 'Package validation failed.' }
$currentPackageFiles = @(Get-RelativeFilePaths -Root $PackagePath)
Remove-StalePackageFilesFromClient `
-ClientPath $ClientPath `
-PreviousFiles $previousPackageFiles `
-CurrentFiles $currentPackageFiles
Remove-MoonWellRuntimeGarbage -Destination $ClientPath
& robocopy $PackagePath $ClientPath /E /R:2 /W:1 /NFL /NDL /NJH /NJS /NP
$robocopyExitCode = $LASTEXITCODE
if ($robocopyExitCode -ge 8) {
@@ -107,8 +134,6 @@ if ($robocopyExitCode -ge 8) {
if (-not $SkipManifest) {
Write-Host '[5/5] Building launcher manifest...'
$python = Get-Command python -ErrorAction SilentlyContinue
if (-not $python) { throw 'Python was not found; launcher manifest was not generated.' }
& $python.Source (Join-Path $repoRoot 'build_manifest.py') `
--dir $PackagePath --output (Join-Path $repoRoot 'manifest.json')
if ($LASTEXITCODE -ne 0) { throw 'Launcher manifest generation failed.' }
+26
View File
@@ -0,0 +1,26 @@
{
"repositoryPatches": [
"Data/patch-4.MPQ",
"Data/patch-Z.MPQ",
"Data/ruRU/patch-ruRU-4.MPQ",
"Data/ruRU/patch-ruRU-5.MPQ",
"Data/ruRU/patch-ruRU-Z.MPQ"
],
"graphicsPatches": [
"Data/patch-X.MPQ",
"Data/ruRU/backup-ruRU.MPQ",
"Data/ruRU/patch-ruRU-6.MPQ",
"Data/ruRU/patch-ruRU-A.mpq",
"Data/ruRU/patch-ruRU-B.mpq",
"Data/ruRU/patch-ruRU-E.mpq",
"Data/ruRU/patch-ruRU-H.MPQ",
"Data/ruRU/patch-ruRU-I.mpq",
"Data/ruRU/patch-ruRU-M.MPQ",
"Data/ruRU/patch-ruRU-S.mpq",
"Data/ruRU/patch-ruRU-T.mpq",
"Data/ruRU/patch-ruRU-U.mpq",
"Data/ruRU/patch-ruRU-W.mpq",
"Data/ruRU/patch-ruRU-X.MPQ",
"Data/ruRU/patch-ruRU-Y.MPQ"
]
}
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
import json
from dataclasses import dataclass
from pathlib import Path
def normalized_path(path: str) -> str:
return path.replace("\\", "/").casefold()
@dataclass(frozen=True)
class PatchLayout:
repository_patches: frozenset[str]
graphics_patches: frozenset[str]
def is_repository_patch(self, path: str) -> bool:
return normalized_path(path) in self.repository_patches
def is_graphics_patch(self, path: str) -> bool:
return normalized_path(path) in self.graphics_patches
def is_external_graphics_patch(self, path: str) -> bool:
key = normalized_path(path)
return key in self.graphics_patches and key not in self.repository_patches
def load_patch_layout(path: Path | None = None) -> PatchLayout:
config_path = path or Path(__file__).resolve().with_name("patch-layout.json")
data = json.loads(config_path.read_text(encoding="utf-8"))
repository = frozenset(normalized_path(item) for item in data["repositoryPatches"])
graphics = frozenset(normalized_path(item) for item in data["graphicsPatches"])
return PatchLayout(repository_patches=repository, graphics_patches=graphics)
PATCH_LAYOUT = load_patch_layout()
+241
View File
@@ -0,0 +1,241 @@
$script:MoonWellExtensionNames = @(
'MoonWell',
'wxl-db2',
'wxl-fdid-moonwell',
'wxl-grasswind',
'wxl-modern-adt',
'wxl-modern-blp',
'wxl-modern-m2',
'wxl-modern-wmo',
'wxl-moonwell-storage-fallback',
'wxl-unit-outline'
)
$patchLayoutPath = Join-Path $PSScriptRoot 'patch-layout.json'
if (-not (Test-Path -LiteralPath $patchLayoutPath -PathType Leaf)) {
throw "Patch layout was not found: $patchLayoutPath"
}
$patchLayout = Get-Content -LiteralPath $patchLayoutPath -Raw -Encoding UTF8 | ConvertFrom-Json
$script:MoonWellRepositoryPatches = @($patchLayout.repositoryPatches)
$script:MoonWellGraphicsPatches = @($patchLayout.graphicsPatches)
function Get-MoonWellExtensionNames {
return @($script:MoonWellExtensionNames)
}
function Get-MoonWellRepositoryPatchPaths {
return @($script:MoonWellRepositoryPatches)
}
function Get-MoonWellGraphicsPatchPaths {
return @($script:MoonWellGraphicsPatches)
}
function Test-MoonWellRepositoryPatch {
param([Parameter(Mandatory)][string]$RelativePath)
foreach ($repositoryPatch in $script:MoonWellRepositoryPatches) {
if ($RelativePath.Replace('/', '\').Equals(
$repositoryPatch.Replace('/', '\'),
[System.StringComparison]::OrdinalIgnoreCase)) {
return $true
}
}
return $false
}
function Test-MoonWellGraphicsPatch {
param([Parameter(Mandatory)][string]$RelativePath)
foreach ($graphicsPatch in $script:MoonWellGraphicsPatches) {
if ($RelativePath.Replace('/', '\').Equals(
$graphicsPatch.Replace('/', '\'),
[System.StringComparison]::OrdinalIgnoreCase)) {
return $true
}
}
return $false
}
function Test-MoonWellPreservedDataPatch {
param([Parameter(Mandatory)][string]$RelativePath)
if (Test-MoonWellRepositoryPatch -RelativePath $RelativePath) {
return $false
}
return Test-MoonWellGraphicsPatch -RelativePath $RelativePath
}
function Assert-SafePackagePath {
param(
[Parameter(Mandatory)]
[string]$PackagePath,
[Parameter(Mandatory)]
[string]$RepositoryPath,
[Parameter()]
[string]$ClientPath
)
$packageRoot = [System.IO.Path]::GetFullPath($PackagePath).TrimEnd('\')
$repositoryRoot = [System.IO.Path]::GetFullPath($RepositoryPath).TrimEnd('\')
$pathRoot = [System.IO.Path]::GetPathRoot($packageRoot).TrimEnd('\')
if (-not $packageRoot -or $packageRoot -eq $pathRoot) {
throw "Refusing to use a filesystem root as package staging: $packageRoot"
}
if ($packageRoot.Equals($repositoryRoot, [System.StringComparison]::OrdinalIgnoreCase) -or
$repositoryRoot.StartsWith($packageRoot + '\', [System.StringComparison]::OrdinalIgnoreCase)) {
throw 'Package staging must not be the repository root or one of its ancestors.'
}
if (-not [string]::IsNullOrWhiteSpace($ClientPath)) {
$clientRoot = [System.IO.Path]::GetFullPath($ClientPath).TrimEnd('\')
$packageIsClientOrChild =
$packageRoot.Equals($clientRoot, [System.StringComparison]::OrdinalIgnoreCase) -or
$packageRoot.StartsWith($clientRoot + '\', [System.StringComparison]::OrdinalIgnoreCase)
$packageContainsClient =
$clientRoot.StartsWith($packageRoot + '\', [System.StringComparison]::OrdinalIgnoreCase)
if ($packageIsClientOrChild -or $packageContainsClient) {
throw 'Package staging must be separate from the installed client tree.'
}
}
}
function Get-RelativeFilePaths {
param([Parameter(Mandatory)][string]$Root)
if (-not (Test-Path -LiteralPath $Root -PathType Container)) {
return @()
}
$absoluteRoot = [System.IO.Path]::GetFullPath($Root).TrimEnd('\')
return @(
Get-ChildItem -LiteralPath $absoluteRoot -Recurse -Force -File |
ForEach-Object { $_.FullName.Substring($absoluteRoot.Length + 1) }
)
}
function Remove-EmptyMoonWellDirectories {
param([Parameter(Mandatory)][string]$Destination)
foreach ($relativeRoot in @('Data\Patch-WXL.MPQ', 'Extensions', 'Utils')) {
$managedRoot = Join-Path $Destination $relativeRoot
if (-not (Test-Path -LiteralPath $managedRoot -PathType Container)) {
continue
}
$directories = @(
Get-ChildItem -LiteralPath $managedRoot -Recurse -Force -Directory |
Sort-Object { $_.FullName.Length } -Descending
)
$directories += Get-Item -LiteralPath $managedRoot
foreach ($directory in $directories) {
if (-not @(Get-ChildItem -LiteralPath $directory.FullName -Force).Count) {
Remove-Item -LiteralPath $directory.FullName -Force
Write-Host "Removed empty managed directory: $($directory.FullName)"
}
}
}
}
function Reset-MoonWellPackageStaging {
param(
[Parameter(Mandatory)][string]$PackagePath,
[Parameter(Mandatory)][string]$RepositoryPath,
[Parameter(Mandatory)][string]$ClientPath
)
Assert-SafePackagePath `
-PackagePath $PackagePath `
-RepositoryPath $RepositoryPath `
-ClientPath $ClientPath
if (Test-Path -LiteralPath $PackagePath) {
Remove-Item -LiteralPath $PackagePath -Recurse -Force
}
New-Item -ItemType Directory -Path $PackagePath -Force | Out-Null
}
function Remove-MoonWellRuntimeGarbage {
param([Parameter(Mandatory)][string]$Destination)
if (-not (Test-Path -LiteralPath $Destination -PathType Container)) {
return
}
$allowedExtensions = @{}
foreach ($name in $script:MoonWellExtensionNames) {
$allowedExtensions[$name.ToLowerInvariant()] = $true
}
$extensionsRoot = Join-Path $Destination 'Extensions'
if (Test-Path -LiteralPath $extensionsRoot -PathType Container) {
foreach ($entry in Get-ChildItem -LiteralPath $extensionsRoot -Force) {
$isAllowedDirectory = $entry.PSIsContainer -and
$allowedExtensions.ContainsKey($entry.Name.ToLowerInvariant())
if (-not $isAllowedDirectory) {
Remove-Item -LiteralPath $entry.FullName -Recurse -Force
Write-Host "Removed unmanaged WarcraftXL extension artifact: $($entry.FullName)"
continue
}
foreach ($dll in Get-ChildItem -LiteralPath $entry.FullName -Force -File -Filter '*.dll') {
$expectedName = "$($entry.Name).dll"
if (-not $dll.Name.Equals($expectedName, [System.StringComparison]::OrdinalIgnoreCase)) {
Remove-Item -LiteralPath $dll.FullName -Force
Write-Host "Removed stale extension DLL: $($dll.FullName)"
}
}
}
}
foreach ($dll in Get-ChildItem -LiteralPath $Destination -Force -File -Filter 'WarcraftXL*.dll') {
if (-not $dll.Name.Equals('WarcraftXL.dll', [System.StringComparison]::OrdinalIgnoreCase)) {
Remove-Item -LiteralPath $dll.FullName -Force
Write-Host "Removed stale WarcraftXL DLL: $($dll.FullName)"
}
}
$utilsRoot = Join-Path $Destination 'Utils'
if (Test-Path -LiteralPath $utilsRoot -PathType Container) {
foreach ($legacyFile in Get-ChildItem -LiteralPath $utilsRoot -Force -File -Filter 'WarcraftXLHost*') {
Remove-Item -LiteralPath $legacyFile.FullName -Force
Write-Host "Removed obsolete WarcraftXL 1.0 host artifact: $($legacyFile.FullName)"
}
}
Remove-EmptyMoonWellDirectories -Destination $Destination
}
function Remove-StalePackageFilesFromClient {
param(
[Parameter(Mandatory)][string]$ClientPath,
[Parameter(Mandatory)][AllowEmptyCollection()][string[]]$PreviousFiles,
[Parameter(Mandatory)][AllowEmptyCollection()][string[]]$CurrentFiles
)
$current = @{}
foreach ($relativePath in $CurrentFiles) {
$current[$relativePath.ToLowerInvariant()] = $true
}
$clientRoot = [System.IO.Path]::GetFullPath($ClientPath).TrimEnd('\')
foreach ($relativePath in $PreviousFiles) {
if ($current.ContainsKey($relativePath.ToLowerInvariant()) -or
(Test-MoonWellPreservedDataPatch -RelativePath $relativePath)) {
continue
}
$target = [System.IO.Path]::GetFullPath((Join-Path $clientRoot $relativePath))
if (-not $target.StartsWith($clientRoot + '\', [System.StringComparison]::OrdinalIgnoreCase)) {
throw "Package inventory contains a path outside the client: $relativePath"
}
if (Test-Path -LiteralPath $target -PathType Leaf) {
Remove-Item -LiteralPath $target -Force
Write-Host "Removed stale packaged file from client: $target"
}
}
Remove-EmptyMoonWellDirectories -Destination $ClientPath
}
+1
View File
@@ -0,0 +1 @@
dvc[s3]==3.67.1
+30 -1
View File
@@ -9,6 +9,7 @@ $ErrorActionPreference = "Stop"
# --- Resolve project root (folder where script is located)
$ROOT = Split-Path -Parent $MyInvocation.MyCommand.Path
. (Join-Path $ROOT 'pipeline-layout.ps1')
# --- Load .env if WOW_HOME not already set
if (-not $env:WOW_HOME) {
@@ -80,6 +81,11 @@ $TOOL = Join-Path $ROOT "tool\target\release\tool.exe"
$RELOAD_SCRIPT = Join-Path $ROOT "reload_wow.bat"
$WXL_BUILD_SCRIPT = Join-Path $ROOT "build-warcraftxl.ps1"
Assert-SafePackagePath `
-PackagePath $DIST_DIR `
-RepositoryPath $ROOT `
-ClientPath $WOW_HOME
function Stop-WowRuntime {
param([Parameter(Mandatory=$true)][string]$ClientPath)
@@ -118,7 +124,14 @@ if (!(Test-Path $TOOL)) {
Pop-Location
}
# --- Build MPQ archives into dist/
# --- Build MPQ archives into a clean dist/
$previousPackageFiles = @(
Get-RelativeFilePaths -Root $DIST_DIR
)
Reset-MoonWellPackageStaging `
-PackagePath $DIST_DIR `
-RepositoryPath $ROOT `
-ClientPath $WOW_HOME
Write-Host "Building MPQ archives from src/ -> dist/..."
& $TOOL $SRC_DIR $DIST_DIR
if ($LASTEXITCODE -ne 0) {
@@ -145,6 +158,22 @@ if ($LASTEXITCODE -ne 0) {
# --- Sync dist/ -> WOW_HOME
Write-Host "Syncing dist/ -> WOW_HOME..."
$python = Get-Command python -ErrorAction SilentlyContinue
if (-not $python) {
Write-Error "Python is required for package validation."
exit 1
}
& $python.Source (Join-Path $ROOT 'validate_package.py') --dir $DIST_DIR
if ($LASTEXITCODE -ne 0) {
Write-Error "Package validation failed!"
exit 1
}
$currentPackageFiles = @(Get-RelativeFilePaths -Root $DIST_DIR)
Remove-StalePackageFilesFromClient `
-ClientPath $WOW_HOME `
-PreviousFiles $previousPackageFiles `
-CurrentFiles $currentPackageFiles
Remove-MoonWellRuntimeGarbage -Destination $WOW_HOME
robocopy $DIST_DIR $WOW_HOME /E /NFL /NDL
# robocopy exit codes 0-7 are success/warnings; 8+ are errors
if ($LASTEXITCODE -ge 8) {
@@ -7,9 +7,6 @@
## Dependencies: CallbackHandler-1.0, AIO_Client, LibStub, Ace3
## X-Hidden: 1
# Libraries
embeds.xml
# Locale
Locale\enUS.lua
Locale\ruRU.lua
Binary file not shown.
+64
View File
@@ -0,0 +1,64 @@
# Sync DBC files from src with files on a server
# Parse .env
if (Test-Path ".env") {
Get-Content ".env" | ForEach-Object {
if ($_ -match '^[^#\s]+=.+') {
$name, $value = $_ -split '=', 2
$name = $name.Trim()
$value = $value.Trim().Trim('"').Trim("'")
[System.Environment]::SetEnvironmentVariable($name, $value)
}
}
} else {
Write-Error ".env not found!"
exit 1
}
$user = $env:SSH_USER
$sshHost = $env:SSH_HOST
$port = $env:SSH_PORT
if (-not $user -or -not $sshHost) {
Write-Error "SSH_USER or SSH_HOST env variables are not defined"
exit 1
}
if (-not $port) {
$port = "22"
}
$input = @(
".\src\Data\ruRU\patch-ruRU-5\DBFilesClient\Achievement.dbc",
".\src\Data\ruRU\patch-ruRU-Z\DBFilesClient\Spell.dbc",
".\src\Data\ruRU\patch-ruRU-Z\DBFilesClient\BarberShopStyle.dbc",
".\src\Data\ruRU\patch-ruRU-Z\DBFilesClient\CreatureDisplayInfoExtra.dbc",
".\src\Data\ruRU\patch-ruRU-Z\DBFilesClient\SpellVisualKitModelAttach.dbc"
)
$output = "/home/mangos/moonwell-server/env/dist/data/dbc"
foreach ($file in $input) {
if (Test-Path $file) {
$fileName = Split-Path $file -Leaf
$outputPath = "$output/$fileName"
Write-Host "Copying $fileName to $output..."
scp -P $port $file "${user}@${sshHost}:${output}"
if ($LASTEXITCODE -eq 0) {
Write-Host "Running chmod..."
ssh -p $port "${user}@${sshHost}" "chown mangos:mangos '$outputPath' && chmod 775 '$outputPath'"
}
} else {
Write-Warning "Input file not found: $file"
}
}
Write-Host "Restarting server..."
ssh -p $port "${user}@${sshHost}" "cd /home/mangos/moonwell-server && docker compose restart ac-worldserver"
Write-Host "Done."
+266
View File
@@ -0,0 +1,266 @@
use std::collections::{HashMap, HashSet};
use std::env;
use std::error::Error;
use std::fs;
use std::path::{Component, Path, PathBuf};
use serde::Serialize;
use wow_mpq::{Archive, special_files::parse_listfile};
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct UnresolvedFile {
name_hash_a: String,
name_hash_b: String,
hash_index: usize,
block_index: usize,
size: u64,
compressed_size: u64,
flags: String,
blob: String,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct UnresolvedManifest {
version: u32,
archive_blocks: usize,
resolved_blocks: usize,
special_blocks: usize,
unresolved: Vec<UnresolvedFile>,
}
fn guess_extension(data: &[u8]) -> &'static str {
match data {
[b'B', b'L', b'P', b'1' | b'2', ..] => "blp",
[b'M', b'D', b'2', b'0' | b'1', ..] => "m2",
[b'S', b'K', b'I', b'N', ..] => "skin",
[b'W', b'D', b'B', b'C', ..] => "dbc",
[b'O', b'g', b'g', b'S', ..] => "ogg",
[b'R', b'I', b'F', b'F', ..] => "wav",
[0x89, b'P', b'N', b'G', ..] => "png",
[b'D', b'D', b'S', b' ', ..] => "dds",
[0xff, 0xd8, 0xff, ..] => "jpg",
_ => "bin",
}
}
fn safe_output_path(root: &Path, archive_path: &str) -> Result<PathBuf, Box<dyn Error>> {
let normalized = archive_path.replace('\\', "/");
let mut relative = PathBuf::new();
for part in normalized.split('/') {
if part.is_empty() || part == "." {
continue;
}
if part == ".." || part.contains(':') {
return Err(format!("unsafe archive path: {archive_path}").into());
}
relative.push(part);
}
if relative.as_os_str().is_empty()
|| relative
.components()
.any(|component| !matches!(component, Component::Normal(_)))
{
return Err(format!("unsafe archive path: {archive_path}").into());
}
Ok(root.join(relative))
}
fn extract_archive(
archive_path: &Path,
output_dir: &Path,
external_listfiles: &[String],
) -> Result<(), Box<dyn Error>> {
if output_dir.exists() && fs::read_dir(output_dir)?.next().is_some() {
return Err(format!(
"output directory is not empty: {} (extract to a new directory)",
output_dir.display()
)
.into());
}
let mut archive = Archive::open(archive_path)?;
let all_entries = archive.list_all()?;
let all_blocks: HashSet<usize> = all_entries
.iter()
.filter_map(|entry| entry.table_indices.and_then(|(_, block)| block))
.collect();
let hashes_by_block: HashMap<usize, (u32, u32)> = archive
.list_all_with_hashes()?
.into_iter()
.filter_map(|entry| Some((entry.table_indices?.1?, entry.hashes?)))
.collect();
let mut special_blocks = HashSet::new();
for special in [
"(listfile)",
"(attributes)",
"(signature)",
"(patch_metadata)",
] {
if let Some(info) = archive.find_file(special)? {
special_blocks.insert(info.block_index);
}
}
let mut files = Vec::new();
if let Ok(listfile) = archive.read_file("(listfile)") {
files.extend(parse_listfile(&listfile)?);
}
for listfile_path in external_listfiles {
files.extend(parse_listfile(&fs::read(listfile_path)?)?);
}
if files.is_empty() {
return Err(format!(
"{} has no readable (listfile); provide an external listfile",
archive_path.display()
)
.into());
}
fs::create_dir_all(output_dir)?;
let mut seen = HashSet::new();
let mut matched_blocks = HashSet::new();
let mut extracted = 0usize;
for archive_name in files {
if archive_name.starts_with('(') && archive_name.ends_with(')') {
continue;
}
// MPQ file lookup is case-insensitive. Avoid writing the same logical path
// twice on Windows when a malformed listfile contains casing duplicates.
if !seen.insert(archive_name.to_ascii_lowercase()) {
continue;
}
let Some(info) = archive.find_file(&archive_name)? else {
continue;
};
if !matched_blocks.insert(info.block_index) {
continue;
}
let output_path = safe_output_path(output_dir, &archive_name)?;
if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent)?;
}
let data = archive
.read_file(&archive_name)
.map_err(|error| format!("cannot extract {archive_name}: {error}"))?;
fs::write(&output_path, data)?;
extracted += 1;
if extracted.is_multiple_of(1000) {
println!("Extracted {extracted} files...");
}
}
println!(
"Extracted {extracted} files from {} to {} ({} data blocks, {} special blocks)",
archive_path.display(),
output_dir.display(),
all_blocks.len(),
special_blocks.len()
);
let unresolved_blocks: Vec<usize> = all_blocks
.difference(&special_blocks)
.filter(|block| !matched_blocks.contains(block))
.copied()
.collect();
if !unresolved_blocks.is_empty() {
let metadata_root = output_dir.join("__mpqmeta__");
let blob_root = metadata_root.join("unresolved");
fs::create_dir_all(&blob_root)?;
let mut unresolved = Vec::with_capacity(unresolved_blocks.len());
for block_index in unresolved_blocks {
let entry = all_entries
.iter()
.find(|entry| entry.table_indices.and_then(|(_, block)| block) == Some(block_index))
.ok_or_else(|| format!("missing metadata for block {block_index}"))?;
let (hash_index, _) = entry
.table_indices
.ok_or_else(|| format!("missing table indices for block {block_index}"))?;
if entry.is_encrypted() || entry.is_patch_file() {
return Err(format!(
"unresolved block {block_index} is encrypted or is a binary patch"
)
.into());
}
let (name_hash_a, name_hash_b) = hashes_by_block
.get(&block_index)
.copied()
.ok_or_else(|| format!("missing name hashes for block {block_index}"))?;
let data = archive.read_file_by_indices(hash_index, Some(block_index))?;
let extension = guess_extension(&data);
let blob_name =
format!("{name_hash_a:08x}-{name_hash_b:08x}-{block_index:08}.{extension}");
fs::write(blob_root.join(&blob_name), data)?;
unresolved.push(UnresolvedFile {
name_hash_a: format!("{name_hash_a:08x}"),
name_hash_b: format!("{name_hash_b:08x}"),
hash_index,
block_index,
size: entry.size,
compressed_size: entry.compressed_size,
flags: format!("0x{:08x}", entry.flags),
blob: format!("unresolved/{blob_name}"),
});
}
unresolved.sort_by_key(|entry| entry.block_index);
let manifest = UnresolvedManifest {
version: 1,
archive_blocks: all_blocks.len(),
resolved_blocks: matched_blocks.len(),
special_blocks: special_blocks.len(),
unresolved,
};
fs::write(
metadata_root.join("unresolved.json"),
serde_json::to_vec_pretty(&manifest)?,
)?;
println!(
"Stored {} path-unresolved data blocks in {}",
manifest.unresolved.len(),
metadata_root.display()
);
}
Ok(())
}
fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().collect();
if args.len() < 3 {
eprintln!("Usage: mpqextract <archive.mpq> <output_dir> [external-listfile ...]");
std::process::exit(2);
}
extract_archive(Path::new(&args[1]), Path::new(&args[2]), &args[3..])
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn keeps_safe_relative_paths() {
let root = Path::new("output");
assert_eq!(
safe_output_path(root, "Character\\Human\\Human.m2").unwrap(),
root.join("Character").join("Human").join("Human.m2")
);
}
#[test]
fn rejects_parent_and_drive_paths() {
assert!(safe_output_path(Path::new("output"), "..\\secret").is_err());
assert!(safe_output_path(Path::new("output"), "C:\\secret").is_err());
}
}
+38
View File
@@ -0,0 +1,38 @@
use std::env;
use std::error::Error;
use std::path::Path;
use wow_mpq::Archive;
fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().collect();
if args.len() != 2 {
eprintln!("Usage: mpqinfo <archive.mpq>");
std::process::exit(2);
}
let archive_path = Path::new(&args[1]);
let mut archive = Archive::open(archive_path)?;
let entries = archive.list()?;
let file_count = entries
.iter()
.filter(|entry| !entry.name.starts_with('('))
.count();
let unpacked_bytes: u64 = entries
.iter()
.filter(|entry| !entry.name.starts_with('('))
.map(|entry| entry.size)
.sum();
let encrypted_files = entries.iter().filter(|entry| entry.is_encrypted()).count();
let patch_files = entries.iter().filter(|entry| entry.is_patch_file()).count();
println!(
"{}\t{}\t{}\t{} encrypted\t{} patch",
archive_path.display(),
file_count,
unpacked_bytes,
encrypted_files,
patch_files
);
Ok(())
}
+78
View File
@@ -0,0 +1,78 @@
use std::collections::BTreeMap;
use std::env;
use std::error::Error;
use std::fs;
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
use wow_mpq::{Archive, special_files::parse_listfile};
fn collect_inputs(path: &Path, inputs: &mut Vec<PathBuf>) -> Result<(), Box<dyn Error>> {
if path.is_file() {
inputs.push(path.to_path_buf());
return Ok(());
}
for entry in WalkDir::new(path) {
let entry = entry?;
if entry.file_type().is_file() {
inputs.push(entry.path().to_path_buf());
}
}
Ok(())
}
fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().collect();
if args.len() < 3 {
eprintln!("Usage: mpqmasterlist <output.txt> <archive-or-listfile> [...]");
std::process::exit(2);
}
let output = Path::new(&args[1]);
let mut inputs = Vec::new();
for input in &args[2..] {
collect_inputs(Path::new(input), &mut inputs)?;
}
let mut names = BTreeMap::new();
let mut sources = 0usize;
for input in inputs {
let extension = input
.extension()
.and_then(|value| value.to_str())
.unwrap_or_default();
let data = if extension.eq_ignore_ascii_case("mpq") {
let mut archive = match Archive::open(&input) {
Ok(archive) => archive,
Err(_) => continue,
};
match archive.read_file("(listfile)") {
Ok(data) => data,
Err(_) => continue,
}
} else if extension.eq_ignore_ascii_case("txt") || extension.eq_ignore_ascii_case("csv") {
fs::read(&input)?
} else {
continue;
};
for name in parse_listfile(&data)? {
names.entry(name.to_ascii_lowercase()).or_insert(name);
}
sources += 1;
}
if let Some(parent) = output.parent() {
fs::create_dir_all(parent)?;
}
let mut contents = names.into_values().collect::<Vec<_>>().join("\r\n");
contents.push_str("\r\n");
fs::write(output, contents)?;
println!(
"Wrote {} unique paths from {sources} listfiles to {}",
parse_listfile(&fs::read(output)?)?.len(),
output.display()
);
Ok(())
}
+87
View File
@@ -0,0 +1,87 @@
use std::env;
use std::error::Error;
use std::fs;
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
use wow_mpq::compression::flags;
use wow_mpq::{ArchiveBuilder, FormatVersion, ListfileOption};
fn collect_files(input_dir: &Path) -> Result<Vec<(PathBuf, String)>, Box<dyn Error>> {
let root = input_dir.canonicalize()?;
let mut files = Vec::new();
for entry in WalkDir::new(&root) {
let entry = entry?;
if !entry.file_type().is_file() {
continue;
}
let source = entry.path().canonicalize()?;
let archive_path = source
.strip_prefix(&root)?
.to_string_lossy()
.replace('\\', "/");
files.push((source, archive_path));
}
files.sort_by(|left, right| {
left.1
.to_ascii_lowercase()
.cmp(&right.1.to_ascii_lowercase())
.then_with(|| left.1.cmp(&right.1))
});
Ok(files)
}
fn pack_directory(input_dir: &Path, output_path: &Path) -> Result<(), Box<dyn Error>> {
if input_dir.join("__mpqmeta__").exists() {
return Err(format!(
"{} contains path-unresolved MPQ blocks and cannot be rebuilt safely",
input_dir.display()
)
.into());
}
let files = collect_files(input_dir)?;
if files.is_empty() {
return Err(format!("input directory contains no files: {}", input_dir.display()).into());
}
if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent)?;
}
let mut builder = ArchiveBuilder::new()
.version(FormatVersion::V2)
.block_size(7)
.default_compression(flags::ZLIB)
.listfile_option(ListfileOption::Generate);
let total = files.len();
for (index, (source, archive_path)) in files.into_iter().enumerate() {
builder = builder.add_file(source, &archive_path);
let packed = index + 1;
if packed.is_multiple_of(1000) {
println!("Queued {packed}/{total} files...");
}
}
builder.build(output_path)?;
println!(
"Packed {total} files from {} to {}",
input_dir.display(),
output_path.display()
);
Ok(())
}
fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().collect();
if args.len() != 3 {
eprintln!("Usage: mpqpack <input_dir> <archive.mpq>");
std::process::exit(2);
}
pack_directory(Path::new(&args[1]), Path::new(&args[2]))
}
-31
View File
@@ -119,35 +119,6 @@ fn stage_loose_assets(
Ok(())
}
fn stage_customization_archives(
project_root: &Path,
output_dir: &Path,
) -> Result<(), Box<dyn Error>> {
let source_dir = project_root.join("new customization mpqs");
let locale_dir = output_dir.join("Data").join("ruRU");
let archives = [
("patch-a-001.mpq", "patch-ruRU-X.MPQ"),
("patch-b-002.mpq", "patch-ruRU-Y.MPQ"),
];
for (source_name, target_name) in archives {
let source = source_dir.join(source_name);
if !source.is_file() {
println!(
"Customization archive not staged (missing): {}",
source.display()
);
continue;
}
fs::create_dir_all(&locale_dir)?;
let target = locale_dir.join(target_name);
fs::copy(&source, &target)?;
println!("Staged customization archive: {}", target.display());
}
Ok(())
}
fn main() -> Result<(), Box<dyn Error>> {
let args: Vec<String> = env::args().collect();
@@ -214,8 +185,6 @@ fn main() -> Result<(), Box<dyn Error>> {
}
}
stage_customization_archives(&project_root, &output_dir)?;
println!("\nAll MPQ archives built successfully.");
Ok(())
}
+167 -16
View File
@@ -11,9 +11,17 @@ import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
from patch_layout import PATCH_LAYOUT, normalized_path
S3_PREFIX = "World of Warcraft"
MANIFEST_S3_KEY = "manifest.json"
MANAGED_MANIFEST_S3_KEY = "moonwell-managed-manifest.json"
LEGACY_MANAGED_FILES = {
"utils/warcraftxlhost.exe",
"utils/warcraftxlhost.log",
"warcraftxl.before-texture-upload-fix.dll",
}
def compute_build_hash(files: list[dict]) -> str:
lines = [
@@ -25,15 +33,59 @@ def compute_build_hash(files: list[dict]) -> str:
def normalized_path_key(path: str) -> str:
"""Match client paths using Windows filesystem semantics."""
return path.replace("\\", "/").casefold()
return normalized_path(path)
def merge_manifests(base_manifest: dict, staging_manifest: dict) -> dict:
def missing_repository_patches(manifest: dict) -> list[str]:
paths = {
normalized_path_key(item["path"])
for item in manifest.get("files", [])
if isinstance(item, dict) and isinstance(item.get("path"), str)
}
return sorted(PATCH_LAYOUT.repository_patches - paths)
def external_graphics_patches(manifest: dict) -> list[str]:
return sorted(
item["path"]
for item in manifest.get("files", [])
if isinstance(item, dict)
and isinstance(item.get("path"), str)
and PATCH_LAYOUT.is_external_graphics_patch(item["path"])
)
def is_legacy_managed_path(path: str) -> bool:
key = normalized_path_key(path)
return (
key in LEGACY_MANAGED_FILES
or key in {"wow.exe", "d3d9.dll", "warcraftxl.dll", "utils/d3d9-native.dll"}
or key.startswith("extensions/")
or key.startswith("data/patch-wxl.mpq/")
)
def merge_manifests(
base_manifest: dict,
staging_manifest: dict,
previous_managed_paths: list[str] | None = None,
) -> dict:
previous_managed_keys = {
normalized_path_key(path) for path in (previous_managed_paths or [])
}
files_by_path = {
normalized_path_key(item["path"]): item
for item in base_manifest.get("files", [])
if not (
normalized_path_key(item["path"]) in previous_managed_keys
or is_legacy_managed_path(item["path"])
or PATCH_LAYOUT.is_repository_patch(item["path"])
)
or PATCH_LAYOUT.is_external_graphics_patch(item["path"])
}
for item in staging_manifest.get("files", []):
if PATCH_LAYOUT.is_external_graphics_patch(item["path"]):
continue
files_by_path[normalized_path_key(item["path"])] = item
files = sorted(files_by_path.values(), key=lambda item: item["path"].casefold())
@@ -43,6 +95,36 @@ def merge_manifests(base_manifest: dict, staging_manifest: dict) -> dict:
}
def find_stale_managed_paths(
base_manifest: dict,
staging_manifest: dict,
previous_managed_paths: list[str] | None = None,
) -> list[str]:
previous_managed_keys = {
normalized_path_key(path) for path in (previous_managed_paths or [])
}
staging_keys = {
normalized_path_key(item["path"])
for item in staging_manifest.get("files", [])
}
return sorted(
(
item["path"]
for item in base_manifest.get("files", [])
if (
(
normalized_path_key(item["path"]) in previous_managed_keys
or is_legacy_managed_path(item["path"])
or PATCH_LAYOUT.is_repository_patch(item["path"])
)
and not PATCH_LAYOUT.is_external_graphics_patch(item["path"])
)
and normalized_path_key(item["path"]) not in staging_keys
),
key=str.casefold,
)
def load_dotenv(env_path: Path) -> dict[str, str]:
values: dict[str, str] = {}
if not env_path.exists():
@@ -103,6 +185,24 @@ def main() -> int:
print(f"ERROR: unable to read local manifest.json: {error}", file=sys.stderr)
return 1
missing_repository = missing_repository_patches(staging_manifest)
if missing_repository:
print(
"ERROR: local manifest is missing repository patches: "
+ ", ".join(missing_repository),
file=sys.stderr,
)
return 1
unexpected_graphics = external_graphics_patches(staging_manifest)
if unexpected_graphics:
print(
"ERROR: managed manifest contains externally managed graphics patches: "
+ ", ".join(unexpected_graphics),
file=sys.stderr,
)
return 1
try:
response = s3.get_object(Bucket=bucket, Key=MANIFEST_S3_KEY)
production_manifest = json.loads(response["Body"].read())
@@ -116,7 +216,32 @@ def main() -> int:
print(f"ERROR: invalid production manifest: {error}", file=sys.stderr)
return 1
merged_manifest = merge_manifests(production_manifest, staging_manifest)
try:
response = s3.get_object(Bucket=bucket, Key=MANAGED_MANIFEST_S3_KEY)
previous_managed_manifest = json.loads(response["Body"].read())
previous_managed_paths = [
item["path"] for item in previous_managed_manifest.get("files", [])
]
except ClientError as error:
error_code = error.response.get("Error", {}).get("Code")
if error_code not in {"NoSuchKey", "404"}:
print(f"ERROR: unable to read managed manifest: {error}", file=sys.stderr)
return 1
previous_managed_paths = []
except (UnicodeDecodeError, json.JSONDecodeError, KeyError) as error:
print(f"ERROR: invalid managed manifest: {error}", file=sys.stderr)
return 1
merged_manifest = merge_manifests(
production_manifest,
staging_manifest,
previous_managed_paths,
)
stale_managed_paths = find_stale_managed_paths(
production_manifest,
staging_manifest,
previous_managed_paths,
)
print(
"Manifest merge: "
f"production={len(production_manifest.get('files', []))}, "
@@ -139,6 +264,9 @@ def main() -> int:
for local_path in files:
rel_path = local_path.relative_to(dist_dir).as_posix()
if PATCH_LAYOUT.is_external_graphics_patch(rel_path):
print(f"Skipping externally managed graphics patch: {rel_path}")
continue
s3_key = f"{S3_PREFIX}/{rel_path}"
size_mb = local_path.stat().st_size / (1024 * 1024)
print(f"Uploading {rel_path} ({size_mb:.1f} MB) -> s3://{bucket}/{s3_key}")
@@ -154,27 +282,50 @@ def main() -> int:
print("\nFinished with errors; production manifest was not changed.")
return 1
manifest_path.write_text(
json.dumps(merged_manifest, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
newline="\n",
json_content_type = "application/json"
no_cache = "no-cache, no-store, must-revalidate"
print(
f"Uploading managed package inventory -> "
f"s3://{bucket}/{MANAGED_MANIFEST_S3_KEY}"
)
try:
s3.put_object(
Bucket=bucket,
Key=MANAGED_MANIFEST_S3_KEY,
Body=(json.dumps(staging_manifest, ensure_ascii=False, indent=2) + "\n").encode("utf-8"),
ContentType=json_content_type,
CacheControl=no_cache,
)
print(" OK")
except Exception as e:
print(f" ERROR: {e}", file=sys.stderr)
return 1
print(f"Uploading manifest.json -> s3://{bucket}/{MANIFEST_S3_KEY}")
try:
s3.upload_file(
str(manifest_path),
bucket,
MANIFEST_S3_KEY,
ExtraArgs={
"ContentType": "application/json",
"CacheControl": "no-cache, no-store, must-revalidate",
},
s3.put_object(
Bucket=bucket,
Key=MANIFEST_S3_KEY,
Body=(json.dumps(merged_manifest, ensure_ascii=False, indent=2) + "\n").encode("utf-8"),
ContentType=json_content_type,
CacheControl=no_cache,
)
print(f" OK")
except Exception as e:
print(f" ERROR: {e}", file=sys.stderr)
errors = True
return 1
if stale_managed_paths:
print(f"Deleting {len(stale_managed_paths)} stale managed object(s)...")
for relative_path in stale_managed_paths:
s3_key = f"{S3_PREFIX}/{relative_path.replace(chr(92), '/')}"
try:
s3.delete_object(Bucket=bucket, Key=s3_key)
print(f" Deleted {s3_key}")
except Exception as e:
print(f" ERROR deleting {s3_key}: {e}", file=sys.stderr)
errors = True
if errors:
print("\nFinished with errors.")
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
import argparse
import sys
from pathlib import Path
from patch_layout import PATCH_LAYOUT, normalized_path
EXTENSIONS = {
"MoonWell",
"wxl-db2",
"wxl-fdid-moonwell",
"wxl-grasswind",
"wxl-modern-adt",
"wxl-modern-blp",
"wxl-modern-m2",
"wxl-modern-wmo",
"wxl-moonwell-storage-fallback",
"wxl-unit-outline",
}
EXTENSION_KEYS = {name.casefold() for name in EXTENSIONS}
ALLOWED_TOP_LEVEL_FILES = {
"d3d9.dll",
"warcraftxl.dll",
"wow.exe",
}
ALLOWED_TOP_LEVEL_DIRS = {"data", "extensions", "utils"}
REQUIRED_RUNTIME_FILES = {
"wow.exe",
"d3d9.dll",
"warcraftxl.dll",
*PATCH_LAYOUT.repository_patches,
"utils/d3d9-native.dll",
*(f"extensions/{name.casefold()}/{name.casefold()}.dll" for name in EXTENSIONS),
}
ALLOWED_DLL_FILES = {
path for path in REQUIRED_RUNTIME_FILES if path.endswith(".dll")
}
BUILD_ONLY_SUFFIXES = {
".bak",
".exp",
".ilk",
".lib",
".log",
".obj",
".old",
".orig",
".pdb",
".pyc",
".rej",
".temp",
".tmp",
}
def normalized(path: Path) -> str:
return normalized_path(path.as_posix())
def validate(root: Path) -> list[str]:
errors: list[str] = []
files = sorted(path for path in root.rglob("*") if path.is_file())
relative_files = {normalized(path.relative_to(root)) for path in files}
for missing in sorted(REQUIRED_RUNTIME_FILES - relative_files):
errors.append(f"missing required runtime file: {missing}")
for entry in root.iterdir():
key = entry.name.casefold()
if entry.is_file() and key not in ALLOWED_TOP_LEVEL_FILES:
errors.append(f"unexpected top-level file: {entry.name}")
elif entry.is_dir() and key not in ALLOWED_TOP_LEVEL_DIRS:
errors.append(f"unexpected top-level directory: {entry.name}")
for path in files:
relative = path.relative_to(root)
relative_key = normalized(relative)
parts = relative.parts
if PATCH_LAYOUT.is_external_graphics_patch(relative.as_posix()):
errors.append(
f"externally managed graphics patch in repository package: {relative.as_posix()}"
)
if path.suffix.casefold() in BUILD_ONLY_SUFFIXES:
errors.append(f"build-only or temporary file: {relative.as_posix()}")
if path.suffix.casefold() == ".dll" and relative_key not in ALLOWED_DLL_FILES:
errors.append(f"unexpected DLL: {relative.as_posix()}")
if path.suffix.casefold() == ".exe" and relative_key != "wow.exe":
errors.append(f"unexpected executable: {relative.as_posix()}")
if parts and parts[0].casefold() == "extensions":
if len(parts) != 3:
errors.append(f"unexpected Extensions layout: {relative.as_posix()}")
continue
extension_name = parts[1]
expected_file = f"{extension_name}.dll"
if extension_name.casefold() not in EXTENSION_KEYS:
errors.append(f"unexpected extension: {relative.as_posix()}")
elif parts[2].casefold() != expected_file.casefold():
errors.append(f"unexpected file in extension package: {relative.as_posix()}")
return errors
def main() -> int:
parser = argparse.ArgumentParser(description="Validate MoonWell package layout")
parser.add_argument("--dir", required=True, help="Package staging directory")
args = parser.parse_args()
root = Path(args.dir).resolve()
if not root.is_dir():
print(f"ERROR: package directory not found: {root}", file=sys.stderr)
return 1
errors = validate(root)
if errors:
print("ERROR: package validation failed:", file=sys.stderr)
for error in errors:
print(f" - {error}", file=sys.stderr)
return 1
print(f"OK: package layout validated ({len(list(root.rglob('*')))} entries)")
return 0
if __name__ == "__main__":
raise SystemExit(main())