dvc

2026-08-22 19:10:11 +04:00
parent ebd76275a1
commit 597f4030a7
30 changed files with 877 additions and 1 deletions
@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache
@@ -0,0 +1,8 @@
+[cache]
+    type = "hardlink,copy"
+[core]
+    remote = customization
+['remote "customization"']
+    url = s3://warcraft-client/dvc/moonwell-client-sources/customization
+    endpointurl = https://storage.yandexcloud.net
+    region = ru-central1
@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore
@@ -2,6 +2,7 @@ WOW_HOME=C:\Games\World of Warcraft 3.3.5

 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
+AWS_SESSION_TOKEN=
 AWS_DEFAULT_REGION=
 AWS_BUCKET=
 AWS_ENDPOINT=
@@ -12,4 +13,4 @@ PTR_REALMLIST=
 LOCAL_REALMLIST=

 SSH_USER=root
-SSH_HOST=strictlypragmatic.ru
+SSH_HOST=strictlypragmatic.ru
@@ -47,6 +47,8 @@ git submodule update --remote vendor/warcraftxl vendor/modules/*
 - Visual Studio 2022 с C++ toolchain для Win32;
 - CMake 3.20+;
 - Rust/Cargo для существующего MPQ-сборщика;
+- Python 3 и DVC с поддержкой S3 для содержимого внешних графических патчей
+  (`py -m pip install -r requirements-dvc.txt`);
 - локальный оригинальный `Wow_Original.exe` с SHA-256
  `AA63A5750D60EF16746C686B3D5E26876D98953EAB08B1C026CD0FAF78E88CB8`.

@@ -128,6 +130,11 @@ WarcraftXL не требуется и в объектное хранилище
 не изменяют `graphicsPatches`. При пересечении списков репозиторный патч имеет
 приоритет.

+Распакованное содержимое больших патчей кастомизации хранится через DVC в
+`assets/graphics/`, а бинарные объекты — в настроенном S3. Инструкции по первичной
+загрузке, изменению и импорту MPQ находятся в
+[`assets/graphics/README.md`](assets/graphics/README.md).
+
 ## Лицензирование

 WarcraftXL распространяется по GPL-3.0 и подключён как отдельный неизменённый upstream submodule.
@@ -0,0 +1 @@
+/patch-X
@@ -0,0 +1,6 @@
+outs:
+- md5: 012ce72d8dfdf089b20a28f6e6158b58.dir
+  size: 1195310
+  nfiles: 4
+  hash: md5
+  path: patch-X
@@ -0,0 +1,14 @@
+/patch-ruRU-X
+/patch-ruRU-Y
+/backup-ruRU
+/patch-ruRU-6
+/patch-ruRU-A
+/patch-ruRU-B
+/patch-ruRU-E
+/patch-ruRU-H
+/patch-ruRU-I
+/patch-ruRU-M
+/patch-ruRU-S
+/patch-ruRU-T
+/patch-ruRU-U
+/patch-ruRU-W
@@ -0,0 +1,6 @@
+outs:
+- md5: b120587dfcd255f78392ca52705a00ff.dir
+  size: 38876467
+  nfiles: 161
+  hash: md5
+  path: backup-ruRU
@@ -0,0 +1,6 @@
+outs:
+- md5: 9ca2ee746d80e8a0c951c5ef476ae654.dir
+  size: 1064210
+  nfiles: 3
+  hash: md5
+  path: patch-ruRU-6
@@ -0,0 +1,6 @@
+outs:
+- md5: 97232ae859eaa1a16b137efd1a2e9122.dir
+  size: 2748988973
+  nfiles: 8817
+  hash: md5
+  path: patch-ruRU-A
@@ -0,0 +1,6 @@
+outs:
+- md5: bb686aebf7c9cae2c02433abde9a9f19.dir
+  size: 771051083
+  nfiles: 2911
+  hash: md5
+  path: patch-ruRU-B
@@ -0,0 +1,6 @@
+outs:
+- md5: d6b2fe0e3f23c47dc944ca4a8f82d97a.dir
+  size: 9275755440
+  nfiles: 21977
+  hash: md5
+  path: patch-ruRU-E
@@ -0,0 +1,6 @@
+outs:
+- md5: 5e95536d2c6f6ff792f05b800105bedc.dir
+  size: 3727395837
+  nfiles: 40296
+  hash: md5
+  path: patch-ruRU-H
@@ -0,0 +1,6 @@
+outs:
+- md5: 5db07768b0814c970d84a99b6072fd9f.dir
+  size: 31453961
+  nfiles: 186
+  hash: md5
+  path: patch-ruRU-I
@@ -0,0 +1,6 @@
+outs:
+- md5: e927ae98c9971f378a82feb3299fdcb8.dir
+  size: 69521651
+  nfiles: 1552
+  hash: md5
+  path: patch-ruRU-M
@@ -0,0 +1,6 @@
+outs:
+- md5: 449383bbe05ebebb6bad2c37da7793c4.dir
+  size: 142167464
+  nfiles: 1751
+  hash: md5
+  path: patch-ruRU-S
@@ -0,0 +1,6 @@
+outs:
+- md5: fc19500b5ce0fd312a415a3118afd58d.dir
+  size: 656858260
+  nfiles: 1810
+  hash: md5
+  path: patch-ruRU-T
@@ -0,0 +1,6 @@
+outs:
+- md5: 783909a6c2ebed5d4f43516fe1ea68bb.dir
+  size: 460642379
+  nfiles: 1837
+  hash: md5
+  path: patch-ruRU-U
@@ -0,0 +1,6 @@
+outs:
+- md5: e2a1448eb0e443797acd577fc4b67249.dir
+  size: 44597910
+  nfiles: 484
+  hash: md5
+  path: patch-ruRU-W
@@ -0,0 +1,6 @@
+outs:
+- md5: 3d322d290959d1ead612f3b76860a1ea.dir
+  size: 3984134962
+  nfiles: 29495
+  hash: md5
+  path: patch-ruRU-X
@@ -0,0 +1,6 @@
+outs:
+- md5: 42f3a8ac35674e33c0a4420c84cadcb8.dir
+  size: 11053212781
+  nfiles: 36191
+  hash: md5
+  path: patch-ruRU-Y
@@ -0,0 +1,66 @@
+# Графические MPQ-патчи
+
+В этой директории через DVC версионируется распакованное содержимое всех 15 патчей из
+`graphicsPatches` в `patch-layout.json`. Git хранит небольшие файлы `*.dvc`, а сами ресурсы —
+в S3 remote `customization`. Хэшированные пути в S3 являются внутренней адресацией DVC:
+исходное дерево с обычными именами восстанавливается командой `dvc pull`.
+
+## Первичная настройка
+
+```powershell
+py -m pip install -r requirements-dvc.txt
+Copy-Item .env.example .env # если локального .env ещё нет
+.\configure-dvc.ps1
+py -m dvc pull
+```
+
+Можно скачать один патч, передав его указатель явно:
+
+```powershell
+py -m dvc pull assets/graphics/Data/ruRU/patch-ruRU-X.dvc
+```
+
+Секреты записываются только в `.dvc/config.local`; этот файл исключён из Git. Вместо ключей
+из `.env` можно использовать AWS profile: `.\configure-dvc.ps1 -Profile moonwell`.
+
+## Изменение ресурсов
+
+DVC-кэш использует hardlink на NTFS. Перед ручным изменением дерева отвяжите его от кэша:
+
+```powershell
+py -m dvc unprotect assets/graphics/Data/ruRU/patch-ruRU-X
+# изменить, добавить или удалить файлы
+py -m dvc add assets/graphics/Data/ruRU/patch-ruRU-X
+py -m dvc push
+git add assets/graphics/Data/ruRU/patch-ruRU-X.dvc
+```
+
+Сначала выполняйте `dvc push`, затем коммитьте Git-указатель, чтобы коммит не ссылался на ещё
+не загруженные объекты.
+
+## Импорт MPQ
+
+`mpqextract` принимает встроенный `(listfile)` и любое количество внешних listfile. Он извлекает
+все известные пути, а блоки с неизвестными именами сохраняет в `__mpqmeta__/unresolved/`.
+Манифест `__mpqmeta__/unresolved.json` связывает каждый blob с MPQ-хэшами и индексом блока,
+поэтому содержимое архива не теряется даже при неполном listfile.
+
+```powershell
+cargo run --release --manifest-path tool/Cargo.toml --bin mpqextract -- `
+  'patch.mpq' 'assets/graphics/Data/ruRU/patch-ruRU-N' 'master-list.txt'
+```
+
+В `patch-ruRU-A`, `B`, `S`, `T`, `U` и `W` есть такие безымянные блоки. Для их безопасной
+пересборки нужен авторитетный listfile с исходными путями; пока он не найден, исходные MPQ в
+дистрибуционном S3 остаются каноническими архивами. `mpqpack` намеренно отказывается собирать
+дерево с `__mpqmeta__`, поскольку подмена путей хэшами изменила бы семантику патча.
+
+Полностью разрешённое дерево собирается так:
+
+```powershell
+cargo run --release --manifest-path tool/Cargo.toml --bin mpqpack -- `
+  'assets/graphics/Data/ruRU/patch-ruRU-X' 'build/graphics/patch-ruRU-X.MPQ'
+```
+
+Сводка исходных объектов, DVC-хэшей и количества разрешённых блоков находится в
+`assets/graphics/sources.json`.
@@ -0,0 +1,157 @@
+{
+  "sourcePrefix": "s3://warcraft-client/World of Warcraft/",
+  "patches": [
+    {
+      "archive": "Data/patch-X.MPQ",
+      "archiveBytes": 496584,
+      "dvcPointer": "assets/graphics/Data/patch-X.dvc",
+      "dvcMd5": "012ce72d8dfdf089b20a28f6e6158b58.dir",
+      "extractedBytes": 1195310,
+      "trackedFiles": 4,
+      "resolvedFiles": 4,
+      "unresolvedBlocks": 0
+    },
+    {
+      "archive": "Data/ruRU/backup-ruRU.MPQ",
+      "archiveBytes": 23981093,
+      "dvcPointer": "assets/graphics/Data/ruRU/backup-ruRU.dvc",
+      "dvcMd5": "b120587dfcd255f78392ca52705a00ff.dir",
+      "extractedBytes": 38876467,
+      "trackedFiles": 161,
+      "resolvedFiles": 161,
+      "unresolvedBlocks": 0
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-6.MPQ",
+      "archiveBytes": 375062,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-6.dvc",
+      "dvcMd5": "9ca2ee746d80e8a0c951c5ef476ae654.dir",
+      "extractedBytes": 1064210,
+      "trackedFiles": 3,
+      "resolvedFiles": 3,
+      "unresolvedBlocks": 0
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-A.mpq",
+      "archiveBytes": 1451338841,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-A.dvc",
+      "dvcMd5": "97232ae859eaa1a16b137efd1a2e9122.dir",
+      "extractedBytes": 2748988973,
+      "trackedFiles": 8817,
+      "resolvedFiles": 5649,
+      "unresolvedBlocks": 3167
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-B.mpq",
+      "archiveBytes": 384938550,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-B.dvc",
+      "dvcMd5": "bb686aebf7c9cae2c02433abde9a9f19.dir",
+      "extractedBytes": 771051083,
+      "trackedFiles": 2911,
+      "resolvedFiles": 1679,
+      "unresolvedBlocks": 1231
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-E.mpq",
+      "archiveBytes": 1862937450,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-E.dvc",
+      "dvcMd5": "d6b2fe0e3f23c47dc944ca4a8f82d97a.dir",
+      "extractedBytes": 9275755440,
+      "trackedFiles": 21977,
+      "resolvedFiles": 21977,
+      "unresolvedBlocks": 0
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-H.MPQ",
+      "archiveBytes": 2154531136,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-H.dvc",
+      "dvcMd5": "5e95536d2c6f6ff792f05b800105bedc.dir",
+      "extractedBytes": 3727395837,
+      "trackedFiles": 40296,
+      "resolvedFiles": 40296,
+      "unresolvedBlocks": 0
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-I.mpq",
+      "archiveBytes": 14891516,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-I.dvc",
+      "dvcMd5": "5db07768b0814c970d84a99b6072fd9f.dir",
+      "extractedBytes": 31453961,
+      "trackedFiles": 186,
+      "resolvedFiles": 186,
+      "unresolvedBlocks": 0
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-M.MPQ",
+      "archiveBytes": 48772900,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-M.dvc",
+      "dvcMd5": "e927ae98c9971f378a82feb3299fdcb8.dir",
+      "extractedBytes": 69521651,
+      "trackedFiles": 1552,
+      "resolvedFiles": 1552,
+      "unresolvedBlocks": 0
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-S.mpq",
+      "archiveBytes": 80492633,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-S.dvc",
+      "dvcMd5": "449383bbe05ebebb6bad2c37da7793c4.dir",
+      "extractedBytes": 142167464,
+      "trackedFiles": 1751,
+      "resolvedFiles": 1432,
+      "unresolvedBlocks": 318
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-T.mpq",
+      "archiveBytes": 418595462,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-T.dvc",
+      "dvcMd5": "fc19500b5ce0fd312a415a3118afd58d.dir",
+      "extractedBytes": 656858260,
+      "trackedFiles": 1810,
+      "resolvedFiles": 1731,
+      "unresolvedBlocks": 78
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-U.mpq",
+      "archiveBytes": 233678341,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-U.dvc",
+      "dvcMd5": "783909a6c2ebed5d4f43516fe1ea68bb.dir",
+      "extractedBytes": 460642379,
+      "trackedFiles": 1837,
+      "resolvedFiles": 1372,
+      "unresolvedBlocks": 464
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-W.mpq",
+      "archiveBytes": 29433979,
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-W.dvc",
+      "dvcMd5": "e2a1448eb0e443797acd577fc4b67249.dir",
+      "extractedBytes": 44597910,
+      "trackedFiles": 484,
+      "resolvedFiles": 381,
+      "unresolvedBlocks": 102
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-X.MPQ",
+      "archiveBytes": 2673854694,
+      "archiveSha256": "CD8CED6FA6C863D509DAE458E6DD44C3986B8B57731B6DC6192FE94125F8C1FD",
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-X.dvc",
+      "dvcMd5": "3d322d290959d1ead612f3b76860a1ea.dir",
+      "extractedBytes": 3984134962,
+      "trackedFiles": 29495,
+      "resolvedFiles": 29495,
+      "unresolvedBlocks": 0
+    },
+    {
+      "archive": "Data/ruRU/patch-ruRU-Y.MPQ",
+      "archiveBytes": 3001665276,
+      "archiveSha256": "534D496C1A1EE023B910ABC80B727BA88BB09CD2F5767100FC18EE41F4306B90",
+      "dvcPointer": "assets/graphics/Data/ruRU/patch-ruRU-Y.dvc",
+      "dvcMd5": "42f3a8ac35674e33c0a4420c84cadcb8.dir",
+      "extractedBytes": 11053212781,
+      "trackedFiles": 36191,
+      "resolvedFiles": 36191,
+      "unresolvedBlocks": 0
+    }
+  ]
+}
@@ -0,0 +1,56 @@
+param(
+    [string]$EnvFile = (Join-Path $PSScriptRoot '.env'),
+    [string]$Profile
+)
+
+$ErrorActionPreference = 'Stop'
+
+function Get-EnvValue([string]$Path, [string]$Name) {
+    if (-not (Test-Path -LiteralPath $Path)) {
+        return $null
+    }
+
+    $line = Get-Content -LiteralPath $Path |
+        Where-Object { $_ -match "^$([regex]::Escape($Name))=" } |
+        Select-Object -First 1
+    if ($line) {
+        return $line.Substring($Name.Length + 1).Trim()
+    }
+    return $null
+}
+
+function Invoke-Dvc([string[]]$Arguments) {
+    & py -m dvc @Arguments
+    if ($LASTEXITCODE -ne 0) {
+        throw "DVC command failed: dvc $($Arguments -join ' ')"
+    }
+}
+
+if (-not (Test-Path -LiteralPath (Join-Path $PSScriptRoot '.dvc\config'))) {
+    throw 'DVC is not initialized in this repository.'
+}
+
+& py -c 'import dvc, dvc_s3' *> $null
+if ($LASTEXITCODE -ne 0) {
+    throw 'DVC with S3 support is not installed. Run: py -m pip install -r requirements-dvc.txt'
+}
+
+if ($Profile) {
+    Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'profile', $Profile)
+} else {
+    $accessKey = Get-EnvValue $EnvFile 'AWS_ACCESS_KEY_ID'
+    $secretKey = Get-EnvValue $EnvFile 'AWS_SECRET_ACCESS_KEY'
+    $sessionToken = Get-EnvValue $EnvFile 'AWS_SESSION_TOKEN'
+
+    if (-not $accessKey -or -not $secretKey) {
+        throw "AWS credentials are missing in $EnvFile. Pass -Profile or fill AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY."
+    }
+
+    Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'access_key_id', $accessKey)
+    Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'secret_access_key', $secretKey)
+    if ($sessionToken) {
+        Invoke-Dvc @('remote', 'modify', '--local', 'customization', 'session_token', $sessionToken)
+    }
+}
+
+Write-Host 'Local DVC credentials configured. They are stored in ignored .dvc/config.local.'
@@ -0,0 +1 @@
+dvc[s3]==3.67.1
@@ -0,0 +1,266 @@
+use std::collections::{HashMap, HashSet};
+use std::env;
+use std::error::Error;
+use std::fs;
+use std::path::{Component, Path, PathBuf};
+
+use serde::Serialize;
+use wow_mpq::{Archive, special_files::parse_listfile};
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct UnresolvedFile {
+    name_hash_a: String,
+    name_hash_b: String,
+    hash_index: usize,
+    block_index: usize,
+    size: u64,
+    compressed_size: u64,
+    flags: String,
+    blob: String,
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct UnresolvedManifest {
+    version: u32,
+    archive_blocks: usize,
+    resolved_blocks: usize,
+    special_blocks: usize,
+    unresolved: Vec<UnresolvedFile>,
+}
+
+fn guess_extension(data: &[u8]) -> &'static str {
+    match data {
+        [b'B', b'L', b'P', b'1' | b'2', ..] => "blp",
+        [b'M', b'D', b'2', b'0' | b'1', ..] => "m2",
+        [b'S', b'K', b'I', b'N', ..] => "skin",
+        [b'W', b'D', b'B', b'C', ..] => "dbc",
+        [b'O', b'g', b'g', b'S', ..] => "ogg",
+        [b'R', b'I', b'F', b'F', ..] => "wav",
+        [0x89, b'P', b'N', b'G', ..] => "png",
+        [b'D', b'D', b'S', b' ', ..] => "dds",
+        [0xff, 0xd8, 0xff, ..] => "jpg",
+        _ => "bin",
+    }
+}
+
+fn safe_output_path(root: &Path, archive_path: &str) -> Result<PathBuf, Box<dyn Error>> {
+    let normalized = archive_path.replace('\\', "/");
+    let mut relative = PathBuf::new();
+
+    for part in normalized.split('/') {
+        if part.is_empty() || part == "." {
+            continue;
+        }
+        if part == ".." || part.contains(':') {
+            return Err(format!("unsafe archive path: {archive_path}").into());
+        }
+        relative.push(part);
+    }
+
+    if relative.as_os_str().is_empty()
+        || relative
+            .components()
+            .any(|component| !matches!(component, Component::Normal(_)))
+    {
+        return Err(format!("unsafe archive path: {archive_path}").into());
+    }
+
+    Ok(root.join(relative))
+}
+
+fn extract_archive(
+    archive_path: &Path,
+    output_dir: &Path,
+    external_listfiles: &[String],
+) -> Result<(), Box<dyn Error>> {
+    if output_dir.exists() && fs::read_dir(output_dir)?.next().is_some() {
+        return Err(format!(
+            "output directory is not empty: {} (extract to a new directory)",
+            output_dir.display()
+        )
+        .into());
+    }
+
+    let mut archive = Archive::open(archive_path)?;
+    let all_entries = archive.list_all()?;
+    let all_blocks: HashSet<usize> = all_entries
+        .iter()
+        .filter_map(|entry| entry.table_indices.and_then(|(_, block)| block))
+        .collect();
+    let hashes_by_block: HashMap<usize, (u32, u32)> = archive
+        .list_all_with_hashes()?
+        .into_iter()
+        .filter_map(|entry| Some((entry.table_indices?.1?, entry.hashes?)))
+        .collect();
+    let mut special_blocks = HashSet::new();
+    for special in [
+        "(listfile)",
+        "(attributes)",
+        "(signature)",
+        "(patch_metadata)",
+    ] {
+        if let Some(info) = archive.find_file(special)? {
+            special_blocks.insert(info.block_index);
+        }
+    }
+
+    let mut files = Vec::new();
+    if let Ok(listfile) = archive.read_file("(listfile)") {
+        files.extend(parse_listfile(&listfile)?);
+    }
+    for listfile_path in external_listfiles {
+        files.extend(parse_listfile(&fs::read(listfile_path)?)?);
+    }
+    if files.is_empty() {
+        return Err(format!(
+            "{} has no readable (listfile); provide an external listfile",
+            archive_path.display()
+        )
+        .into());
+    }
+
+    fs::create_dir_all(output_dir)?;
+    let mut seen = HashSet::new();
+    let mut matched_blocks = HashSet::new();
+    let mut extracted = 0usize;
+
+    for archive_name in files {
+        if archive_name.starts_with('(') && archive_name.ends_with(')') {
+            continue;
+        }
+
+        // MPQ file lookup is case-insensitive. Avoid writing the same logical path
+        // twice on Windows when a malformed listfile contains casing duplicates.
+        if !seen.insert(archive_name.to_ascii_lowercase()) {
+            continue;
+        }
+
+        let Some(info) = archive.find_file(&archive_name)? else {
+            continue;
+        };
+        if !matched_blocks.insert(info.block_index) {
+            continue;
+        }
+
+        let output_path = safe_output_path(output_dir, &archive_name)?;
+        if let Some(parent) = output_path.parent() {
+            fs::create_dir_all(parent)?;
+        }
+
+        let data = archive
+            .read_file(&archive_name)
+            .map_err(|error| format!("cannot extract {archive_name}: {error}"))?;
+        fs::write(&output_path, data)?;
+        extracted += 1;
+
+        if extracted.is_multiple_of(1000) {
+            println!("Extracted {extracted} files...");
+        }
+    }
+
+    println!(
+        "Extracted {extracted} files from {} to {} ({} data blocks, {} special blocks)",
+        archive_path.display(),
+        output_dir.display(),
+        all_blocks.len(),
+        special_blocks.len()
+    );
+
+    let unresolved_blocks: Vec<usize> = all_blocks
+        .difference(&special_blocks)
+        .filter(|block| !matched_blocks.contains(block))
+        .copied()
+        .collect();
+    if !unresolved_blocks.is_empty() {
+        let metadata_root = output_dir.join("__mpqmeta__");
+        let blob_root = metadata_root.join("unresolved");
+        fs::create_dir_all(&blob_root)?;
+        let mut unresolved = Vec::with_capacity(unresolved_blocks.len());
+
+        for block_index in unresolved_blocks {
+            let entry = all_entries
+                .iter()
+                .find(|entry| entry.table_indices.and_then(|(_, block)| block) == Some(block_index))
+                .ok_or_else(|| format!("missing metadata for block {block_index}"))?;
+            let (hash_index, _) = entry
+                .table_indices
+                .ok_or_else(|| format!("missing table indices for block {block_index}"))?;
+            if entry.is_encrypted() || entry.is_patch_file() {
+                return Err(format!(
+                    "unresolved block {block_index} is encrypted or is a binary patch"
+                )
+                .into());
+            }
+            let (name_hash_a, name_hash_b) = hashes_by_block
+                .get(&block_index)
+                .copied()
+                .ok_or_else(|| format!("missing name hashes for block {block_index}"))?;
+            let data = archive.read_file_by_indices(hash_index, Some(block_index))?;
+            let extension = guess_extension(&data);
+            let blob_name =
+                format!("{name_hash_a:08x}-{name_hash_b:08x}-{block_index:08}.{extension}");
+            fs::write(blob_root.join(&blob_name), data)?;
+            unresolved.push(UnresolvedFile {
+                name_hash_a: format!("{name_hash_a:08x}"),
+                name_hash_b: format!("{name_hash_b:08x}"),
+                hash_index,
+                block_index,
+                size: entry.size,
+                compressed_size: entry.compressed_size,
+                flags: format!("0x{:08x}", entry.flags),
+                blob: format!("unresolved/{blob_name}"),
+            });
+        }
+
+        unresolved.sort_by_key(|entry| entry.block_index);
+        let manifest = UnresolvedManifest {
+            version: 1,
+            archive_blocks: all_blocks.len(),
+            resolved_blocks: matched_blocks.len(),
+            special_blocks: special_blocks.len(),
+            unresolved,
+        };
+        fs::write(
+            metadata_root.join("unresolved.json"),
+            serde_json::to_vec_pretty(&manifest)?,
+        )?;
+        println!(
+            "Stored {} path-unresolved data blocks in {}",
+            manifest.unresolved.len(),
+            metadata_root.display()
+        );
+    }
+    Ok(())
+}
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let args: Vec<String> = env::args().collect();
+    if args.len() < 3 {
+        eprintln!("Usage: mpqextract <archive.mpq> <output_dir> [external-listfile ...]");
+        std::process::exit(2);
+    }
+
+    extract_archive(Path::new(&args[1]), Path::new(&args[2]), &args[3..])
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn keeps_safe_relative_paths() {
+        let root = Path::new("output");
+        assert_eq!(
+            safe_output_path(root, "Character\\Human\\Human.m2").unwrap(),
+            root.join("Character").join("Human").join("Human.m2")
+        );
+    }
+
+    #[test]
+    fn rejects_parent_and_drive_paths() {
+        assert!(safe_output_path(Path::new("output"), "..\\secret").is_err());
+        assert!(safe_output_path(Path::new("output"), "C:\\secret").is_err());
+    }
+}
@@ -0,0 +1,38 @@
+use std::env;
+use std::error::Error;
+use std::path::Path;
+
+use wow_mpq::Archive;
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let args: Vec<String> = env::args().collect();
+    if args.len() != 2 {
+        eprintln!("Usage: mpqinfo <archive.mpq>");
+        std::process::exit(2);
+    }
+
+    let archive_path = Path::new(&args[1]);
+    let mut archive = Archive::open(archive_path)?;
+    let entries = archive.list()?;
+    let file_count = entries
+        .iter()
+        .filter(|entry| !entry.name.starts_with('('))
+        .count();
+    let unpacked_bytes: u64 = entries
+        .iter()
+        .filter(|entry| !entry.name.starts_with('('))
+        .map(|entry| entry.size)
+        .sum();
+    let encrypted_files = entries.iter().filter(|entry| entry.is_encrypted()).count();
+    let patch_files = entries.iter().filter(|entry| entry.is_patch_file()).count();
+
+    println!(
+        "{}\t{}\t{}\t{} encrypted\t{} patch",
+        archive_path.display(),
+        file_count,
+        unpacked_bytes,
+        encrypted_files,
+        patch_files
+    );
+    Ok(())
+}
@@ -0,0 +1,78 @@
+use std::collections::BTreeMap;
+use std::env;
+use std::error::Error;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use walkdir::WalkDir;
+use wow_mpq::{Archive, special_files::parse_listfile};
+
+fn collect_inputs(path: &Path, inputs: &mut Vec<PathBuf>) -> Result<(), Box<dyn Error>> {
+    if path.is_file() {
+        inputs.push(path.to_path_buf());
+        return Ok(());
+    }
+
+    for entry in WalkDir::new(path) {
+        let entry = entry?;
+        if entry.file_type().is_file() {
+            inputs.push(entry.path().to_path_buf());
+        }
+    }
+    Ok(())
+}
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let args: Vec<String> = env::args().collect();
+    if args.len() < 3 {
+        eprintln!("Usage: mpqmasterlist <output.txt> <archive-or-listfile> [...]");
+        std::process::exit(2);
+    }
+
+    let output = Path::new(&args[1]);
+    let mut inputs = Vec::new();
+    for input in &args[2..] {
+        collect_inputs(Path::new(input), &mut inputs)?;
+    }
+
+    let mut names = BTreeMap::new();
+    let mut sources = 0usize;
+    for input in inputs {
+        let extension = input
+            .extension()
+            .and_then(|value| value.to_str())
+            .unwrap_or_default();
+        let data = if extension.eq_ignore_ascii_case("mpq") {
+            let mut archive = match Archive::open(&input) {
+                Ok(archive) => archive,
+                Err(_) => continue,
+            };
+            match archive.read_file("(listfile)") {
+                Ok(data) => data,
+                Err(_) => continue,
+            }
+        } else if extension.eq_ignore_ascii_case("txt") || extension.eq_ignore_ascii_case("csv") {
+            fs::read(&input)?
+        } else {
+            continue;
+        };
+
+        for name in parse_listfile(&data)? {
+            names.entry(name.to_ascii_lowercase()).or_insert(name);
+        }
+        sources += 1;
+    }
+
+    if let Some(parent) = output.parent() {
+        fs::create_dir_all(parent)?;
+    }
+    let mut contents = names.into_values().collect::<Vec<_>>().join("\r\n");
+    contents.push_str("\r\n");
+    fs::write(output, contents)?;
+    println!(
+        "Wrote {} unique paths from {sources} listfiles to {}",
+        parse_listfile(&fs::read(output)?)?.len(),
+        output.display()
+    );
+    Ok(())
+}
@@ -0,0 +1,87 @@
+use std::env;
+use std::error::Error;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use walkdir::WalkDir;
+use wow_mpq::compression::flags;
+use wow_mpq::{ArchiveBuilder, FormatVersion, ListfileOption};
+
+fn collect_files(input_dir: &Path) -> Result<Vec<(PathBuf, String)>, Box<dyn Error>> {
+    let root = input_dir.canonicalize()?;
+    let mut files = Vec::new();
+
+    for entry in WalkDir::new(&root) {
+        let entry = entry?;
+        if !entry.file_type().is_file() {
+            continue;
+        }
+
+        let source = entry.path().canonicalize()?;
+        let archive_path = source
+            .strip_prefix(&root)?
+            .to_string_lossy()
+            .replace('\\', "/");
+        files.push((source, archive_path));
+    }
+
+    files.sort_by(|left, right| {
+        left.1
+            .to_ascii_lowercase()
+            .cmp(&right.1.to_ascii_lowercase())
+            .then_with(|| left.1.cmp(&right.1))
+    });
+    Ok(files)
+}
+
+fn pack_directory(input_dir: &Path, output_path: &Path) -> Result<(), Box<dyn Error>> {
+    if input_dir.join("__mpqmeta__").exists() {
+        return Err(format!(
+            "{} contains path-unresolved MPQ blocks and cannot be rebuilt safely",
+            input_dir.display()
+        )
+        .into());
+    }
+
+    let files = collect_files(input_dir)?;
+    if files.is_empty() {
+        return Err(format!("input directory contains no files: {}", input_dir.display()).into());
+    }
+
+    if let Some(parent) = output_path.parent() {
+        fs::create_dir_all(parent)?;
+    }
+
+    let mut builder = ArchiveBuilder::new()
+        .version(FormatVersion::V2)
+        .block_size(7)
+        .default_compression(flags::ZLIB)
+        .listfile_option(ListfileOption::Generate);
+
+    let total = files.len();
+    for (index, (source, archive_path)) in files.into_iter().enumerate() {
+        builder = builder.add_file(source, &archive_path);
+        let packed = index + 1;
+        if packed.is_multiple_of(1000) {
+            println!("Queued {packed}/{total} files...");
+        }
+    }
+
+    builder.build(output_path)?;
+    println!(
+        "Packed {total} files from {} to {}",
+        input_dir.display(),
+        output_path.display()
+    );
+    Ok(())
+}
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let args: Vec<String> = env::args().collect();
+    if args.len() != 3 {
+        eprintln!("Usage: mpqpack <input_dir> <archive.mpq>");
+        std::process::exit(2);
+    }
+
+    pack_directory(Path::new(&args[1]), Path::new(&args[2]))
+}