diff --git a/.dvc/.gitignore b/.dvc/.gitignore new file mode 100644 index 0000000..528f30c --- /dev/null +++ b/.dvc/.gitignore @@ -0,0 +1,3 @@ +/config.local +/tmp +/cache diff --git a/.dvc/config b/.dvc/config new file mode 100644 index 0000000..de06c63 --- /dev/null +++ b/.dvc/config @@ -0,0 +1,8 @@ +[cache] + type = "hardlink,copy" +[core] + remote = customization +['remote "customization"'] + url = s3://warcraft-client/dvc/moonwell-client-sources/customization + endpointurl = https://storage.yandexcloud.net + region = ru-central1 diff --git a/.dvcignore b/.dvcignore new file mode 100644 index 0000000..5197305 --- /dev/null +++ b/.dvcignore @@ -0,0 +1,3 @@ +# Add patterns of files dvc should ignore, which could improve +# the performance. Learn more at +# https://dvc.org/doc/user-guide/dvcignore diff --git a/.env.example b/.env.example index 127fb09..ecd9902 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,7 @@ WOW_HOME=C:\Games\World of Warcraft 3.3.5 AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= +AWS_SESSION_TOKEN= AWS_DEFAULT_REGION= AWS_BUCKET= AWS_ENDPOINT= @@ -12,4 +13,4 @@ PTR_REALMLIST= LOCAL_REALMLIST= SSH_USER=root -SSH_HOST=strictlypragmatic.ru \ No newline at end of file +SSH_HOST=strictlypragmatic.ru diff --git a/README.md b/README.md index 03fe75b..3925b66 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ git submodule update --remote vendor/warcraftxl vendor/modules/* - Visual Studio 2022 с C++ toolchain для Win32; - CMake 3.20+; - Rust/Cargo для существующего MPQ-сборщика; +- Python 3 и DVC с поддержкой S3 для содержимого внешних графических патчей + (`py -m pip install -r requirements-dvc.txt`); - локальный оригинальный `Wow_Original.exe` с SHA-256 `AA63A5750D60EF16746C686B3D5E26876D98953EAB08B1C026CD0FAF78E88CB8`. @@ -128,6 +130,11 @@ WarcraftXL не требуется и в объектное хранилище не изменяют `graphicsPatches`. При пересечении списков репозиторный патч имеет приоритет. +Распакованное содержимое больших патчей кастомизации хранится через DVC в +`assets/graphics/`, а бинарные объекты — в настроенном S3. Инструкции по первичной +загрузке, изменению и импорту MPQ находятся в +[`assets/graphics/README.md`](assets/graphics/README.md). + ## Лицензирование WarcraftXL распространяется по GPL-3.0 и подключён как отдельный неизменённый upstream submodule. diff --git a/assets/graphics/Data/.gitignore b/assets/graphics/Data/.gitignore new file mode 100644 index 0000000..319d01e --- /dev/null +++ b/assets/graphics/Data/.gitignore @@ -0,0 +1 @@ +/patch-X diff --git a/assets/graphics/Data/patch-X.dvc b/assets/graphics/Data/patch-X.dvc new file mode 100644 index 0000000..1ae71dd --- /dev/null +++ b/assets/graphics/Data/patch-X.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 012ce72d8dfdf089b20a28f6e6158b58.dir + size: 1195310 + nfiles: 4 + hash: md5 + path: patch-X diff --git a/assets/graphics/Data/ruRU/.gitignore b/assets/graphics/Data/ruRU/.gitignore new file mode 100644 index 0000000..d878d28 --- /dev/null +++ b/assets/graphics/Data/ruRU/.gitignore @@ -0,0 +1,14 @@ +/patch-ruRU-X +/patch-ruRU-Y +/backup-ruRU +/patch-ruRU-6 +/patch-ruRU-A +/patch-ruRU-B +/patch-ruRU-E +/patch-ruRU-H +/patch-ruRU-I +/patch-ruRU-M +/patch-ruRU-S +/patch-ruRU-T +/patch-ruRU-U +/patch-ruRU-W diff --git a/assets/graphics/Data/ruRU/backup-ruRU.dvc b/assets/graphics/Data/ruRU/backup-ruRU.dvc new file mode 100644 index 0000000..06b88ec --- /dev/null +++ b/assets/graphics/Data/ruRU/backup-ruRU.dvc @@ -0,0 +1,6 @@ +outs: +- md5: b120587dfcd255f78392ca52705a00ff.dir + size: 38876467 + nfiles: 161 + hash: md5 + path: backup-ruRU diff --git a/assets/graphics/Data/ruRU/patch-ruRU-6.dvc b/assets/graphics/Data/ruRU/patch-ruRU-6.dvc new file mode 100644 index 0000000..d96100e --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-6.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 9ca2ee746d80e8a0c951c5ef476ae654.dir + size: 1064210 + nfiles: 3 + hash: md5 + path: patch-ruRU-6 diff --git a/assets/graphics/Data/ruRU/patch-ruRU-A.dvc b/assets/graphics/Data/ruRU/patch-ruRU-A.dvc new file mode 100644 index 0000000..4f882bd --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-A.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 97232ae859eaa1a16b137efd1a2e9122.dir + size: 2748988973 + nfiles: 8817 + hash: md5 + path: patch-ruRU-A diff --git a/assets/graphics/Data/ruRU/patch-ruRU-B.dvc b/assets/graphics/Data/ruRU/patch-ruRU-B.dvc new file mode 100644 index 0000000..92d084f --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-B.dvc @@ -0,0 +1,6 @@ +outs: +- md5: bb686aebf7c9cae2c02433abde9a9f19.dir + size: 771051083 + nfiles: 2911 + hash: md5 + path: patch-ruRU-B diff --git a/assets/graphics/Data/ruRU/patch-ruRU-E.dvc b/assets/graphics/Data/ruRU/patch-ruRU-E.dvc new file mode 100644 index 0000000..9ffa1b9 --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-E.dvc @@ -0,0 +1,6 @@ +outs: +- md5: d6b2fe0e3f23c47dc944ca4a8f82d97a.dir + size: 9275755440 + nfiles: 21977 + hash: md5 + path: patch-ruRU-E diff --git a/assets/graphics/Data/ruRU/patch-ruRU-H.dvc b/assets/graphics/Data/ruRU/patch-ruRU-H.dvc new file mode 100644 index 0000000..bd5d9d8 --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-H.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 5e95536d2c6f6ff792f05b800105bedc.dir + size: 3727395837 + nfiles: 40296 + hash: md5 + path: patch-ruRU-H diff --git a/assets/graphics/Data/ruRU/patch-ruRU-I.dvc b/assets/graphics/Data/ruRU/patch-ruRU-I.dvc new file mode 100644 index 0000000..6db23ce --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-I.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 5db07768b0814c970d84a99b6072fd9f.dir + size: 31453961 + nfiles: 186 + hash: md5 + path: patch-ruRU-I diff --git a/assets/graphics/Data/ruRU/patch-ruRU-M.dvc b/assets/graphics/Data/ruRU/patch-ruRU-M.dvc new file mode 100644 index 0000000..c5c33e3 --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-M.dvc @@ -0,0 +1,6 @@ +outs: +- md5: e927ae98c9971f378a82feb3299fdcb8.dir + size: 69521651 + nfiles: 1552 + hash: md5 + path: patch-ruRU-M diff --git a/assets/graphics/Data/ruRU/patch-ruRU-S.dvc b/assets/graphics/Data/ruRU/patch-ruRU-S.dvc new file mode 100644 index 0000000..5b437eb --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-S.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 449383bbe05ebebb6bad2c37da7793c4.dir + size: 142167464 + nfiles: 1751 + hash: md5 + path: patch-ruRU-S diff --git a/assets/graphics/Data/ruRU/patch-ruRU-T.dvc b/assets/graphics/Data/ruRU/patch-ruRU-T.dvc new file mode 100644 index 0000000..57b9e41 --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-T.dvc @@ -0,0 +1,6 @@ +outs: +- md5: fc19500b5ce0fd312a415a3118afd58d.dir + size: 656858260 + nfiles: 1810 + hash: md5 + path: patch-ruRU-T diff --git a/assets/graphics/Data/ruRU/patch-ruRU-U.dvc b/assets/graphics/Data/ruRU/patch-ruRU-U.dvc new file mode 100644 index 0000000..0e6b522 --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-U.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 783909a6c2ebed5d4f43516fe1ea68bb.dir + size: 460642379 + nfiles: 1837 + hash: md5 + path: patch-ruRU-U diff --git a/assets/graphics/Data/ruRU/patch-ruRU-W.dvc b/assets/graphics/Data/ruRU/patch-ruRU-W.dvc new file mode 100644 index 0000000..2765d42 --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-W.dvc @@ -0,0 +1,6 @@ +outs: +- md5: e2a1448eb0e443797acd577fc4b67249.dir + size: 44597910 + nfiles: 484 + hash: md5 + path: patch-ruRU-W diff --git a/assets/graphics/Data/ruRU/patch-ruRU-X.dvc b/assets/graphics/Data/ruRU/patch-ruRU-X.dvc new file mode 100644 index 0000000..94d4577 --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-X.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 3d322d290959d1ead612f3b76860a1ea.dir + size: 3984134962 + nfiles: 29495 + hash: md5 + path: patch-ruRU-X diff --git a/assets/graphics/Data/ruRU/patch-ruRU-Y.dvc b/assets/graphics/Data/ruRU/patch-ruRU-Y.dvc new file mode 100644 index 0000000..f023ca6 --- /dev/null +++ b/assets/graphics/Data/ruRU/patch-ruRU-Y.dvc @@ -0,0 +1,6 @@ +outs: +- md5: 42f3a8ac35674e33c0a4420c84cadcb8.dir + size: 11053212781 + nfiles: 36191 + hash: md5 + path: patch-ruRU-Y diff --git a/assets/graphics/README.md b/assets/graphics/README.md new file mode 100644 index 0000000..0910380 --- /dev/null +++ b/assets/graphics/README.md @@ -0,0 +1,66 @@ +# Графические MPQ-патчи + +В этой директории через DVC версионируется распакованное содержимое всех 15 патчей из +`graphicsPatches` в `patch-layout.json`. Git хранит небольшие файлы `*.dvc`, а сами ресурсы — +в S3 remote `customization`. Хэшированные пути в S3 являются внутренней адресацией DVC: +исходное дерево с обычными именами восстанавливается командой `dvc pull`. + +## Первичная настройка + +```powershell +py -m pip install -r requirements-dvc.txt +Copy-Item .env.example .env # если локального .env ещё нет +.\configure-dvc.ps1 +py -m dvc pull +``` + +Можно скачать один патч, передав его указатель явно: + +```powershell +py -m dvc pull assets/graphics/Data/ruRU/patch-ruRU-X.dvc +``` + +Секреты записываются только в `.dvc/config.local`; этот файл исключён из Git. Вместо ключей +из `.env` можно использовать AWS profile: `.\configure-dvc.ps1 -Profile moonwell`. + +## Изменение ресурсов + +DVC-кэш использует hardlink на NTFS. Перед ручным изменением дерева отвяжите его от кэша: + +```powershell +py -m dvc unprotect assets/graphics/Data/ruRU/patch-ruRU-X +# изменить, добавить или удалить файлы +py -m dvc add assets/graphics/Data/ruRU/patch-ruRU-X +py -m dvc push +git add assets/graphics/Data/ruRU/patch-ruRU-X.dvc +``` + +Сначала выполняйте `dvc push`, затем коммитьте Git-указатель, чтобы коммит не ссылался на ещё +не загруженные объекты. + +## Импорт MPQ + +`mpqextract` принимает встроенный `(listfile)` и любое количество внешних listfile. Он извлекает +все известные пути, а блоки с неизвестными именами сохраняет в `__mpqmeta__/unresolved/`. +Манифест `__mpqmeta__/unresolved.json` связывает каждый blob с MPQ-хэшами и индексом блока, +поэтому содержимое архива не теряется даже при неполном listfile. + +```powershell +cargo run --release --manifest-path tool/Cargo.toml --bin mpqextract -- ` + 'patch.mpq' 'assets/graphics/Data/ruRU/patch-ruRU-N' 'master-list.txt' +``` + +В `patch-ruRU-A`, `B`, `S`, `T`, `U` и `W` есть такие безымянные блоки. Для их безопасной +пересборки нужен авторитетный listfile с исходными путями; пока он не найден, исходные MPQ в +дистрибуционном S3 остаются каноническими архивами. `mpqpack` намеренно отказывается собирать +дерево с `__mpqmeta__`, поскольку подмена путей хэшами изменила бы семантику патча. + +Полностью разрешённое дерево собирается так: + +```powershell +cargo run --release --manifest-path tool/Cargo.toml --bin mpqpack -- ` + 'assets/graphics/Data/ruRU/patch-ruRU-X' 'build/graphics/patch-ruRU-X.MPQ' +``` + +Сводка исходных объектов, DVC-хэшей и количества разрешённых блоков находится в +`assets/graphics/sources.json`. diff --git a/assets/graphics/sources.json b/assets/graphics/sources.json new file mode 100644 index 0000000..2441b15 --- /dev/null +++ b/assets/graphics/sources.json @@ -0,0 +1,157 @@ +{ + "sourcePrefix": "s3://warcraft-client/World of Warcraft/", + "patches": [ + { + "archive": "Data/patch-X.MPQ", + "archiveBytes": 496584, + "dvcPointer": "assets/graphics/Data/patch-X.dvc", + "dvcMd5": "012ce72d8dfdf089b20a28f6e6158b58.dir", + "extractedBytes": 1195310, + "trackedFiles": 4, + "resolvedFiles": 4, + "unresolvedBlocks": 0 + }, + { + "archive": "Data/ruRU/backup-ruRU.MPQ", + "archiveBytes": 23981093, + "dvcPointer": "assets/graphics/Data/ruRU/backup-ruRU.dvc", + "dvcMd5": "b120587dfcd255f78392ca52705a00ff.dir", + "extractedBytes": 38876467, + "trackedFiles": 161, + "resolvedFiles": 161, + "unresolvedBlocks": 0 + }, + { + "archive": "Data/ruRU/patch-ruRU-6.MPQ", + "archiveBytes": 375062, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-6.dvc", + "dvcMd5": "9ca2ee746d80e8a0c951c5ef476ae654.dir", + "extractedBytes": 1064210, + "trackedFiles": 3, + "resolvedFiles": 3, + "unresolvedBlocks": 0 + }, + { + "archive": "Data/ruRU/patch-ruRU-A.mpq", + "archiveBytes": 1451338841, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-A.dvc", + "dvcMd5": "97232ae859eaa1a16b137efd1a2e9122.dir", + "extractedBytes": 2748988973, + "trackedFiles": 8817, + "resolvedFiles": 5649, + "unresolvedBlocks": 3167 + }, + { + "archive": "Data/ruRU/patch-ruRU-B.mpq", + "archiveBytes": 384938550, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-B.dvc", + "dvcMd5": "bb686aebf7c9cae2c02433abde9a9f19.dir", + "extractedBytes": 771051083, + "trackedFiles": 2911, + "resolvedFiles": 1679, + "unresolvedBlocks": 1231 + }, + { + "archive": "Data/ruRU/patch-ruRU-E.mpq", + "archiveBytes": 1862937450, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-E.dvc", + "dvcMd5": "d6b2fe0e3f23c47dc944ca4a8f82d97a.dir", + "extractedBytes": 9275755440, + "trackedFiles": 21977, + "resolvedFiles": 21977, + "unresolvedBlocks": 0 + }, + { + "archive": "Data/ruRU/patch-ruRU-H.MPQ", + "archiveBytes": 2154531136, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-H.dvc", + "dvcMd5": "5e95536d2c6f6ff792f05b800105bedc.dir", + "extractedBytes": 3727395837, + "trackedFiles": 40296, + "resolvedFiles": 40296, + "unresolvedBlocks": 0 + }, + { + "archive": "Data/ruRU/patch-ruRU-I.mpq", + "archiveBytes": 14891516, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-I.dvc", + "dvcMd5": "5db07768b0814c970d84a99b6072fd9f.dir", + "extractedBytes": 31453961, + "trackedFiles": 186, + "resolvedFiles": 186, + "unresolvedBlocks": 0 + }, + { + "archive": "Data/ruRU/patch-ruRU-M.MPQ", + "archiveBytes": 48772900, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-M.dvc", + "dvcMd5": "e927ae98c9971f378a82feb3299fdcb8.dir", + "extractedBytes": 69521651, + "trackedFiles": 1552, + "resolvedFiles": 1552, + "unresolvedBlocks": 0 + }, + { + "archive": "Data/ruRU/patch-ruRU-S.mpq", + "archiveBytes": 80492633, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-S.dvc", + "dvcMd5": "449383bbe05ebebb6bad2c37da7793c4.dir", + "extractedBytes": 142167464, + "trackedFiles": 1751, + "resolvedFiles": 1432, + "unresolvedBlocks": 318 + }, + { + "archive": "Data/ruRU/patch-ruRU-T.mpq", + "archiveBytes": 418595462, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-T.dvc", + "dvcMd5": "fc19500b5ce0fd312a415a3118afd58d.dir", + "extractedBytes": 656858260, + "trackedFiles": 1810, + "resolvedFiles": 1731, + "unresolvedBlocks": 78 + }, + { + "archive": "Data/ruRU/patch-ruRU-U.mpq", + "archiveBytes": 233678341, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-U.dvc", + "dvcMd5": "783909a6c2ebed5d4f43516fe1ea68bb.dir", + "extractedBytes": 460642379, + "trackedFiles": 1837, + "resolvedFiles": 1372, + "unresolvedBlocks": 464 + }, + { + "archive": "Data/ruRU/patch-ruRU-W.mpq", + "archiveBytes": 29433979, + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-W.dvc", + "dvcMd5": "e2a1448eb0e443797acd577fc4b67249.dir", + "extractedBytes": 44597910, + "trackedFiles": 484, + "resolvedFiles": 381, + "unresolvedBlocks": 102 + }, + { + "archive": "Data/ruRU/patch-ruRU-X.MPQ", + "archiveBytes": 2673854694, + "archiveSha256": "CD8CED6FA6C863D509DAE458E6DD44C3986B8B57731B6DC6192FE94125F8C1FD", + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-X.dvc", + "dvcMd5": "3d322d290959d1ead612f3b76860a1ea.dir", + "extractedBytes": 3984134962, + "trackedFiles": 29495, + "resolvedFiles": 29495, + "unresolvedBlocks": 0 + }, + { + "archive": "Data/ruRU/patch-ruRU-Y.MPQ", + "archiveBytes": 3001665276, + "archiveSha256": "534D496C1A1EE023B910ABC80B727BA88BB09CD2F5767100FC18EE41F4306B90", + "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-Y.dvc", + "dvcMd5": "42f3a8ac35674e33c0a4420c84cadcb8.dir", + "extractedBytes": 11053212781, + "trackedFiles": 36191, + "resolvedFiles": 36191, + "unresolvedBlocks": 0 + } + ] +} diff --git a/configure-dvc.ps1 b/configure-dvc.ps1 new file mode 100644 index 0000000..aa1677f --- /dev/null +++ b/configure-dvc.ps1 @@ -0,0 +1,56 @@ +param( + [string]$EnvFile = (Join-Path $PSScriptRoot '.env'), + [string]$Profile +) + +$ErrorActionPreference = 'Stop' + +function Get-EnvValue([string]$Path, [string]$Name) { + if (-not (Test-Path -LiteralPath $Path)) { + return $null + } + + $line = Get-Content -LiteralPath $Path | + Where-Object { $_ -match "^$([regex]::Escape($Name))=" } | + Select-Object -First 1 + if ($line) { + return $line.Substring($Name.Length + 1).Trim() + } + return $null +} + +function Invoke-Dvc([string[]]$Arguments) { + & py -m dvc @Arguments + if ($LASTEXITCODE -ne 0) { + throw "DVC command failed: dvc $($Arguments -join ' ')" + } +} + +if (-not (Test-Path -LiteralPath (Join-Path $PSScriptRoot '.dvc\config'))) { + throw 'DVC is not initialized in this repository.' +} + +& py -c 'import dvc, dvc_s3' *> $null +if ($LASTEXITCODE -ne 0) { + throw 'DVC with S3 support is not installed. Run: py -m pip install -r requirements-dvc.txt' +} + +if ($Profile) { + Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'profile', $Profile) +} else { + $accessKey = Get-EnvValue $EnvFile 'AWS_ACCESS_KEY_ID' + $secretKey = Get-EnvValue $EnvFile 'AWS_SECRET_ACCESS_KEY' + $sessionToken = Get-EnvValue $EnvFile 'AWS_SESSION_TOKEN' + + if (-not $accessKey -or -not $secretKey) { + throw "AWS credentials are missing in $EnvFile. Pass -Profile or fill AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY." + } + + Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'access_key_id', $accessKey) + Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'secret_access_key', $secretKey) + if ($sessionToken) { + Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'session_token', $sessionToken) + } +} + +Write-Host 'Local DVC credentials configured. They are stored in ignored .dvc/config.local.' diff --git a/requirements-dvc.txt b/requirements-dvc.txt new file mode 100644 index 0000000..27084dd --- /dev/null +++ b/requirements-dvc.txt @@ -0,0 +1 @@ +dvc[s3]==3.67.1 diff --git a/tool/src/bin/mpqextract.rs b/tool/src/bin/mpqextract.rs new file mode 100644 index 0000000..3ec1b75 --- /dev/null +++ b/tool/src/bin/mpqextract.rs @@ -0,0 +1,266 @@ +use std::collections::{HashMap, HashSet}; +use std::env; +use std::error::Error; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +use serde::Serialize; +use wow_mpq::{Archive, special_files::parse_listfile}; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct UnresolvedFile { + name_hash_a: String, + name_hash_b: String, + hash_index: usize, + block_index: usize, + size: u64, + compressed_size: u64, + flags: String, + blob: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct UnresolvedManifest { + version: u32, + archive_blocks: usize, + resolved_blocks: usize, + special_blocks: usize, + unresolved: Vec, +} + +fn guess_extension(data: &[u8]) -> &'static str { + match data { + [b'B', b'L', b'P', b'1' | b'2', ..] => "blp", + [b'M', b'D', b'2', b'0' | b'1', ..] => "m2", + [b'S', b'K', b'I', b'N', ..] => "skin", + [b'W', b'D', b'B', b'C', ..] => "dbc", + [b'O', b'g', b'g', b'S', ..] => "ogg", + [b'R', b'I', b'F', b'F', ..] => "wav", + [0x89, b'P', b'N', b'G', ..] => "png", + [b'D', b'D', b'S', b' ', ..] => "dds", + [0xff, 0xd8, 0xff, ..] => "jpg", + _ => "bin", + } +} + +fn safe_output_path(root: &Path, archive_path: &str) -> Result> { + let normalized = archive_path.replace('\\', "/"); + let mut relative = PathBuf::new(); + + for part in normalized.split('/') { + if part.is_empty() || part == "." { + continue; + } + if part == ".." || part.contains(':') { + return Err(format!("unsafe archive path: {archive_path}").into()); + } + relative.push(part); + } + + if relative.as_os_str().is_empty() + || relative + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(format!("unsafe archive path: {archive_path}").into()); + } + + Ok(root.join(relative)) +} + +fn extract_archive( + archive_path: &Path, + output_dir: &Path, + external_listfiles: &[String], +) -> Result<(), Box> { + if output_dir.exists() && fs::read_dir(output_dir)?.next().is_some() { + return Err(format!( + "output directory is not empty: {} (extract to a new directory)", + output_dir.display() + ) + .into()); + } + + let mut archive = Archive::open(archive_path)?; + let all_entries = archive.list_all()?; + let all_blocks: HashSet = all_entries + .iter() + .filter_map(|entry| entry.table_indices.and_then(|(_, block)| block)) + .collect(); + let hashes_by_block: HashMap = archive + .list_all_with_hashes()? + .into_iter() + .filter_map(|entry| Some((entry.table_indices?.1?, entry.hashes?))) + .collect(); + let mut special_blocks = HashSet::new(); + for special in [ + "(listfile)", + "(attributes)", + "(signature)", + "(patch_metadata)", + ] { + if let Some(info) = archive.find_file(special)? { + special_blocks.insert(info.block_index); + } + } + + let mut files = Vec::new(); + if let Ok(listfile) = archive.read_file("(listfile)") { + files.extend(parse_listfile(&listfile)?); + } + for listfile_path in external_listfiles { + files.extend(parse_listfile(&fs::read(listfile_path)?)?); + } + if files.is_empty() { + return Err(format!( + "{} has no readable (listfile); provide an external listfile", + archive_path.display() + ) + .into()); + } + + fs::create_dir_all(output_dir)?; + let mut seen = HashSet::new(); + let mut matched_blocks = HashSet::new(); + let mut extracted = 0usize; + + for archive_name in files { + if archive_name.starts_with('(') && archive_name.ends_with(')') { + continue; + } + + // MPQ file lookup is case-insensitive. Avoid writing the same logical path + // twice on Windows when a malformed listfile contains casing duplicates. + if !seen.insert(archive_name.to_ascii_lowercase()) { + continue; + } + + let Some(info) = archive.find_file(&archive_name)? else { + continue; + }; + if !matched_blocks.insert(info.block_index) { + continue; + } + + let output_path = safe_output_path(output_dir, &archive_name)?; + if let Some(parent) = output_path.parent() { + fs::create_dir_all(parent)?; + } + + let data = archive + .read_file(&archive_name) + .map_err(|error| format!("cannot extract {archive_name}: {error}"))?; + fs::write(&output_path, data)?; + extracted += 1; + + if extracted.is_multiple_of(1000) { + println!("Extracted {extracted} files..."); + } + } + + println!( + "Extracted {extracted} files from {} to {} ({} data blocks, {} special blocks)", + archive_path.display(), + output_dir.display(), + all_blocks.len(), + special_blocks.len() + ); + + let unresolved_blocks: Vec = all_blocks + .difference(&special_blocks) + .filter(|block| !matched_blocks.contains(block)) + .copied() + .collect(); + if !unresolved_blocks.is_empty() { + let metadata_root = output_dir.join("__mpqmeta__"); + let blob_root = metadata_root.join("unresolved"); + fs::create_dir_all(&blob_root)?; + let mut unresolved = Vec::with_capacity(unresolved_blocks.len()); + + for block_index in unresolved_blocks { + let entry = all_entries + .iter() + .find(|entry| entry.table_indices.and_then(|(_, block)| block) == Some(block_index)) + .ok_or_else(|| format!("missing metadata for block {block_index}"))?; + let (hash_index, _) = entry + .table_indices + .ok_or_else(|| format!("missing table indices for block {block_index}"))?; + if entry.is_encrypted() || entry.is_patch_file() { + return Err(format!( + "unresolved block {block_index} is encrypted or is a binary patch" + ) + .into()); + } + let (name_hash_a, name_hash_b) = hashes_by_block + .get(&block_index) + .copied() + .ok_or_else(|| format!("missing name hashes for block {block_index}"))?; + let data = archive.read_file_by_indices(hash_index, Some(block_index))?; + let extension = guess_extension(&data); + let blob_name = + format!("{name_hash_a:08x}-{name_hash_b:08x}-{block_index:08}.{extension}"); + fs::write(blob_root.join(&blob_name), data)?; + unresolved.push(UnresolvedFile { + name_hash_a: format!("{name_hash_a:08x}"), + name_hash_b: format!("{name_hash_b:08x}"), + hash_index, + block_index, + size: entry.size, + compressed_size: entry.compressed_size, + flags: format!("0x{:08x}", entry.flags), + blob: format!("unresolved/{blob_name}"), + }); + } + + unresolved.sort_by_key(|entry| entry.block_index); + let manifest = UnresolvedManifest { + version: 1, + archive_blocks: all_blocks.len(), + resolved_blocks: matched_blocks.len(), + special_blocks: special_blocks.len(), + unresolved, + }; + fs::write( + metadata_root.join("unresolved.json"), + serde_json::to_vec_pretty(&manifest)?, + )?; + println!( + "Stored {} path-unresolved data blocks in {}", + manifest.unresolved.len(), + metadata_root.display() + ); + } + Ok(()) +} + +fn main() -> Result<(), Box> { + let args: Vec = env::args().collect(); + if args.len() < 3 { + eprintln!("Usage: mpqextract [external-listfile ...]"); + std::process::exit(2); + } + + extract_archive(Path::new(&args[1]), Path::new(&args[2]), &args[3..]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_safe_relative_paths() { + let root = Path::new("output"); + assert_eq!( + safe_output_path(root, "Character\\Human\\Human.m2").unwrap(), + root.join("Character").join("Human").join("Human.m2") + ); + } + + #[test] + fn rejects_parent_and_drive_paths() { + assert!(safe_output_path(Path::new("output"), "..\\secret").is_err()); + assert!(safe_output_path(Path::new("output"), "C:\\secret").is_err()); + } +} diff --git a/tool/src/bin/mpqinfo.rs b/tool/src/bin/mpqinfo.rs new file mode 100644 index 0000000..b528e34 --- /dev/null +++ b/tool/src/bin/mpqinfo.rs @@ -0,0 +1,38 @@ +use std::env; +use std::error::Error; +use std::path::Path; + +use wow_mpq::Archive; + +fn main() -> Result<(), Box> { + let args: Vec = env::args().collect(); + if args.len() != 2 { + eprintln!("Usage: mpqinfo "); + std::process::exit(2); + } + + let archive_path = Path::new(&args[1]); + let mut archive = Archive::open(archive_path)?; + let entries = archive.list()?; + let file_count = entries + .iter() + .filter(|entry| !entry.name.starts_with('(')) + .count(); + let unpacked_bytes: u64 = entries + .iter() + .filter(|entry| !entry.name.starts_with('(')) + .map(|entry| entry.size) + .sum(); + let encrypted_files = entries.iter().filter(|entry| entry.is_encrypted()).count(); + let patch_files = entries.iter().filter(|entry| entry.is_patch_file()).count(); + + println!( + "{}\t{}\t{}\t{} encrypted\t{} patch", + archive_path.display(), + file_count, + unpacked_bytes, + encrypted_files, + patch_files + ); + Ok(()) +} diff --git a/tool/src/bin/mpqmasterlist.rs b/tool/src/bin/mpqmasterlist.rs new file mode 100644 index 0000000..bbe2fea --- /dev/null +++ b/tool/src/bin/mpqmasterlist.rs @@ -0,0 +1,78 @@ +use std::collections::BTreeMap; +use std::env; +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use walkdir::WalkDir; +use wow_mpq::{Archive, special_files::parse_listfile}; + +fn collect_inputs(path: &Path, inputs: &mut Vec) -> Result<(), Box> { + if path.is_file() { + inputs.push(path.to_path_buf()); + return Ok(()); + } + + for entry in WalkDir::new(path) { + let entry = entry?; + if entry.file_type().is_file() { + inputs.push(entry.path().to_path_buf()); + } + } + Ok(()) +} + +fn main() -> Result<(), Box> { + let args: Vec = env::args().collect(); + if args.len() < 3 { + eprintln!("Usage: mpqmasterlist [...]"); + std::process::exit(2); + } + + let output = Path::new(&args[1]); + let mut inputs = Vec::new(); + for input in &args[2..] { + collect_inputs(Path::new(input), &mut inputs)?; + } + + let mut names = BTreeMap::new(); + let mut sources = 0usize; + for input in inputs { + let extension = input + .extension() + .and_then(|value| value.to_str()) + .unwrap_or_default(); + let data = if extension.eq_ignore_ascii_case("mpq") { + let mut archive = match Archive::open(&input) { + Ok(archive) => archive, + Err(_) => continue, + }; + match archive.read_file("(listfile)") { + Ok(data) => data, + Err(_) => continue, + } + } else if extension.eq_ignore_ascii_case("txt") || extension.eq_ignore_ascii_case("csv") { + fs::read(&input)? + } else { + continue; + }; + + for name in parse_listfile(&data)? { + names.entry(name.to_ascii_lowercase()).or_insert(name); + } + sources += 1; + } + + if let Some(parent) = output.parent() { + fs::create_dir_all(parent)?; + } + let mut contents = names.into_values().collect::>().join("\r\n"); + contents.push_str("\r\n"); + fs::write(output, contents)?; + println!( + "Wrote {} unique paths from {sources} listfiles to {}", + parse_listfile(&fs::read(output)?)?.len(), + output.display() + ); + Ok(()) +} diff --git a/tool/src/bin/mpqpack.rs b/tool/src/bin/mpqpack.rs new file mode 100644 index 0000000..d0cd1ca --- /dev/null +++ b/tool/src/bin/mpqpack.rs @@ -0,0 +1,87 @@ +use std::env; +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use walkdir::WalkDir; +use wow_mpq::compression::flags; +use wow_mpq::{ArchiveBuilder, FormatVersion, ListfileOption}; + +fn collect_files(input_dir: &Path) -> Result, Box> { + let root = input_dir.canonicalize()?; + let mut files = Vec::new(); + + for entry in WalkDir::new(&root) { + let entry = entry?; + if !entry.file_type().is_file() { + continue; + } + + let source = entry.path().canonicalize()?; + let archive_path = source + .strip_prefix(&root)? + .to_string_lossy() + .replace('\\', "/"); + files.push((source, archive_path)); + } + + files.sort_by(|left, right| { + left.1 + .to_ascii_lowercase() + .cmp(&right.1.to_ascii_lowercase()) + .then_with(|| left.1.cmp(&right.1)) + }); + Ok(files) +} + +fn pack_directory(input_dir: &Path, output_path: &Path) -> Result<(), Box> { + if input_dir.join("__mpqmeta__").exists() { + return Err(format!( + "{} contains path-unresolved MPQ blocks and cannot be rebuilt safely", + input_dir.display() + ) + .into()); + } + + let files = collect_files(input_dir)?; + if files.is_empty() { + return Err(format!("input directory contains no files: {}", input_dir.display()).into()); + } + + if let Some(parent) = output_path.parent() { + fs::create_dir_all(parent)?; + } + + let mut builder = ArchiveBuilder::new() + .version(FormatVersion::V2) + .block_size(7) + .default_compression(flags::ZLIB) + .listfile_option(ListfileOption::Generate); + + let total = files.len(); + for (index, (source, archive_path)) in files.into_iter().enumerate() { + builder = builder.add_file(source, &archive_path); + let packed = index + 1; + if packed.is_multiple_of(1000) { + println!("Queued {packed}/{total} files..."); + } + } + + builder.build(output_path)?; + println!( + "Packed {total} files from {} to {}", + input_dir.display(), + output_path.display() + ); + Ok(()) +} + +fn main() -> Result<(), Box> { + let args: Vec = env::args().collect(); + if args.len() != 3 { + eprintln!("Usage: mpqpack "); + std::process::exit(2); + } + + pack_directory(Path::new(&args[1]), Path::new(&args[2])) +}