dvc
This commit is contained in:
@@ -0,0 +1,266 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::env;
|
||||
use std::error::Error;
|
||||
use std::fs;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use serde::Serialize;
|
||||
use wow_mpq::{Archive, special_files::parse_listfile};
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct UnresolvedFile {
|
||||
name_hash_a: String,
|
||||
name_hash_b: String,
|
||||
hash_index: usize,
|
||||
block_index: usize,
|
||||
size: u64,
|
||||
compressed_size: u64,
|
||||
flags: String,
|
||||
blob: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct UnresolvedManifest {
|
||||
version: u32,
|
||||
archive_blocks: usize,
|
||||
resolved_blocks: usize,
|
||||
special_blocks: usize,
|
||||
unresolved: Vec<UnresolvedFile>,
|
||||
}
|
||||
|
||||
fn guess_extension(data: &[u8]) -> &'static str {
|
||||
match data {
|
||||
[b'B', b'L', b'P', b'1' | b'2', ..] => "blp",
|
||||
[b'M', b'D', b'2', b'0' | b'1', ..] => "m2",
|
||||
[b'S', b'K', b'I', b'N', ..] => "skin",
|
||||
[b'W', b'D', b'B', b'C', ..] => "dbc",
|
||||
[b'O', b'g', b'g', b'S', ..] => "ogg",
|
||||
[b'R', b'I', b'F', b'F', ..] => "wav",
|
||||
[0x89, b'P', b'N', b'G', ..] => "png",
|
||||
[b'D', b'D', b'S', b' ', ..] => "dds",
|
||||
[0xff, 0xd8, 0xff, ..] => "jpg",
|
||||
_ => "bin",
|
||||
}
|
||||
}
|
||||
|
||||
fn safe_output_path(root: &Path, archive_path: &str) -> Result<PathBuf, Box<dyn Error>> {
|
||||
let normalized = archive_path.replace('\\', "/");
|
||||
let mut relative = PathBuf::new();
|
||||
|
||||
for part in normalized.split('/') {
|
||||
if part.is_empty() || part == "." {
|
||||
continue;
|
||||
}
|
||||
if part == ".." || part.contains(':') {
|
||||
return Err(format!("unsafe archive path: {archive_path}").into());
|
||||
}
|
||||
relative.push(part);
|
||||
}
|
||||
|
||||
if relative.as_os_str().is_empty()
|
||||
|| relative
|
||||
.components()
|
||||
.any(|component| !matches!(component, Component::Normal(_)))
|
||||
{
|
||||
return Err(format!("unsafe archive path: {archive_path}").into());
|
||||
}
|
||||
|
||||
Ok(root.join(relative))
|
||||
}
|
||||
|
||||
fn extract_archive(
|
||||
archive_path: &Path,
|
||||
output_dir: &Path,
|
||||
external_listfiles: &[String],
|
||||
) -> Result<(), Box<dyn Error>> {
|
||||
if output_dir.exists() && fs::read_dir(output_dir)?.next().is_some() {
|
||||
return Err(format!(
|
||||
"output directory is not empty: {} (extract to a new directory)",
|
||||
output_dir.display()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
let mut archive = Archive::open(archive_path)?;
|
||||
let all_entries = archive.list_all()?;
|
||||
let all_blocks: HashSet<usize> = all_entries
|
||||
.iter()
|
||||
.filter_map(|entry| entry.table_indices.and_then(|(_, block)| block))
|
||||
.collect();
|
||||
let hashes_by_block: HashMap<usize, (u32, u32)> = archive
|
||||
.list_all_with_hashes()?
|
||||
.into_iter()
|
||||
.filter_map(|entry| Some((entry.table_indices?.1?, entry.hashes?)))
|
||||
.collect();
|
||||
let mut special_blocks = HashSet::new();
|
||||
for special in [
|
||||
"(listfile)",
|
||||
"(attributes)",
|
||||
"(signature)",
|
||||
"(patch_metadata)",
|
||||
] {
|
||||
if let Some(info) = archive.find_file(special)? {
|
||||
special_blocks.insert(info.block_index);
|
||||
}
|
||||
}
|
||||
|
||||
let mut files = Vec::new();
|
||||
if let Ok(listfile) = archive.read_file("(listfile)") {
|
||||
files.extend(parse_listfile(&listfile)?);
|
||||
}
|
||||
for listfile_path in external_listfiles {
|
||||
files.extend(parse_listfile(&fs::read(listfile_path)?)?);
|
||||
}
|
||||
if files.is_empty() {
|
||||
return Err(format!(
|
||||
"{} has no readable (listfile); provide an external listfile",
|
||||
archive_path.display()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
fs::create_dir_all(output_dir)?;
|
||||
let mut seen = HashSet::new();
|
||||
let mut matched_blocks = HashSet::new();
|
||||
let mut extracted = 0usize;
|
||||
|
||||
for archive_name in files {
|
||||
if archive_name.starts_with('(') && archive_name.ends_with(')') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// MPQ file lookup is case-insensitive. Avoid writing the same logical path
|
||||
// twice on Windows when a malformed listfile contains casing duplicates.
|
||||
if !seen.insert(archive_name.to_ascii_lowercase()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(info) = archive.find_file(&archive_name)? else {
|
||||
continue;
|
||||
};
|
||||
if !matched_blocks.insert(info.block_index) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let output_path = safe_output_path(output_dir, &archive_name)?;
|
||||
if let Some(parent) = output_path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
let data = archive
|
||||
.read_file(&archive_name)
|
||||
.map_err(|error| format!("cannot extract {archive_name}: {error}"))?;
|
||||
fs::write(&output_path, data)?;
|
||||
extracted += 1;
|
||||
|
||||
if extracted.is_multiple_of(1000) {
|
||||
println!("Extracted {extracted} files...");
|
||||
}
|
||||
}
|
||||
|
||||
println!(
|
||||
"Extracted {extracted} files from {} to {} ({} data blocks, {} special blocks)",
|
||||
archive_path.display(),
|
||||
output_dir.display(),
|
||||
all_blocks.len(),
|
||||
special_blocks.len()
|
||||
);
|
||||
|
||||
let unresolved_blocks: Vec<usize> = all_blocks
|
||||
.difference(&special_blocks)
|
||||
.filter(|block| !matched_blocks.contains(block))
|
||||
.copied()
|
||||
.collect();
|
||||
if !unresolved_blocks.is_empty() {
|
||||
let metadata_root = output_dir.join("__mpqmeta__");
|
||||
let blob_root = metadata_root.join("unresolved");
|
||||
fs::create_dir_all(&blob_root)?;
|
||||
let mut unresolved = Vec::with_capacity(unresolved_blocks.len());
|
||||
|
||||
for block_index in unresolved_blocks {
|
||||
let entry = all_entries
|
||||
.iter()
|
||||
.find(|entry| entry.table_indices.and_then(|(_, block)| block) == Some(block_index))
|
||||
.ok_or_else(|| format!("missing metadata for block {block_index}"))?;
|
||||
let (hash_index, _) = entry
|
||||
.table_indices
|
||||
.ok_or_else(|| format!("missing table indices for block {block_index}"))?;
|
||||
if entry.is_encrypted() || entry.is_patch_file() {
|
||||
return Err(format!(
|
||||
"unresolved block {block_index} is encrypted or is a binary patch"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let (name_hash_a, name_hash_b) = hashes_by_block
|
||||
.get(&block_index)
|
||||
.copied()
|
||||
.ok_or_else(|| format!("missing name hashes for block {block_index}"))?;
|
||||
let data = archive.read_file_by_indices(hash_index, Some(block_index))?;
|
||||
let extension = guess_extension(&data);
|
||||
let blob_name =
|
||||
format!("{name_hash_a:08x}-{name_hash_b:08x}-{block_index:08}.{extension}");
|
||||
fs::write(blob_root.join(&blob_name), data)?;
|
||||
unresolved.push(UnresolvedFile {
|
||||
name_hash_a: format!("{name_hash_a:08x}"),
|
||||
name_hash_b: format!("{name_hash_b:08x}"),
|
||||
hash_index,
|
||||
block_index,
|
||||
size: entry.size,
|
||||
compressed_size: entry.compressed_size,
|
||||
flags: format!("0x{:08x}", entry.flags),
|
||||
blob: format!("unresolved/{blob_name}"),
|
||||
});
|
||||
}
|
||||
|
||||
unresolved.sort_by_key(|entry| entry.block_index);
|
||||
let manifest = UnresolvedManifest {
|
||||
version: 1,
|
||||
archive_blocks: all_blocks.len(),
|
||||
resolved_blocks: matched_blocks.len(),
|
||||
special_blocks: special_blocks.len(),
|
||||
unresolved,
|
||||
};
|
||||
fs::write(
|
||||
metadata_root.join("unresolved.json"),
|
||||
serde_json::to_vec_pretty(&manifest)?,
|
||||
)?;
|
||||
println!(
|
||||
"Stored {} path-unresolved data blocks in {}",
|
||||
manifest.unresolved.len(),
|
||||
metadata_root.display()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() < 3 {
|
||||
eprintln!("Usage: mpqextract <archive.mpq> <output_dir> [external-listfile ...]");
|
||||
std::process::exit(2);
|
||||
}
|
||||
|
||||
extract_archive(Path::new(&args[1]), Path::new(&args[2]), &args[3..])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn keeps_safe_relative_paths() {
|
||||
let root = Path::new("output");
|
||||
assert_eq!(
|
||||
safe_output_path(root, "Character\\Human\\Human.m2").unwrap(),
|
||||
root.join("Character").join("Human").join("Human.m2")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_parent_and_drive_paths() {
|
||||
assert!(safe_output_path(Path::new("output"), "..\\secret").is_err());
|
||||
assert!(safe_output_path(Path::new("output"), "C:\\secret").is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
use std::env;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
|
||||
use wow_mpq::Archive;
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() != 2 {
|
||||
eprintln!("Usage: mpqinfo <archive.mpq>");
|
||||
std::process::exit(2);
|
||||
}
|
||||
|
||||
let archive_path = Path::new(&args[1]);
|
||||
let mut archive = Archive::open(archive_path)?;
|
||||
let entries = archive.list()?;
|
||||
let file_count = entries
|
||||
.iter()
|
||||
.filter(|entry| !entry.name.starts_with('('))
|
||||
.count();
|
||||
let unpacked_bytes: u64 = entries
|
||||
.iter()
|
||||
.filter(|entry| !entry.name.starts_with('('))
|
||||
.map(|entry| entry.size)
|
||||
.sum();
|
||||
let encrypted_files = entries.iter().filter(|entry| entry.is_encrypted()).count();
|
||||
let patch_files = entries.iter().filter(|entry| entry.is_patch_file()).count();
|
||||
|
||||
println!(
|
||||
"{}\t{}\t{}\t{} encrypted\t{} patch",
|
||||
archive_path.display(),
|
||||
file_count,
|
||||
unpacked_bytes,
|
||||
encrypted_files,
|
||||
patch_files
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::env;
|
||||
use std::error::Error;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use walkdir::WalkDir;
|
||||
use wow_mpq::{Archive, special_files::parse_listfile};
|
||||
|
||||
fn collect_inputs(path: &Path, inputs: &mut Vec<PathBuf>) -> Result<(), Box<dyn Error>> {
|
||||
if path.is_file() {
|
||||
inputs.push(path.to_path_buf());
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for entry in WalkDir::new(path) {
|
||||
let entry = entry?;
|
||||
if entry.file_type().is_file() {
|
||||
inputs.push(entry.path().to_path_buf());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() < 3 {
|
||||
eprintln!("Usage: mpqmasterlist <output.txt> <archive-or-listfile> [...]");
|
||||
std::process::exit(2);
|
||||
}
|
||||
|
||||
let output = Path::new(&args[1]);
|
||||
let mut inputs = Vec::new();
|
||||
for input in &args[2..] {
|
||||
collect_inputs(Path::new(input), &mut inputs)?;
|
||||
}
|
||||
|
||||
let mut names = BTreeMap::new();
|
||||
let mut sources = 0usize;
|
||||
for input in inputs {
|
||||
let extension = input
|
||||
.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.unwrap_or_default();
|
||||
let data = if extension.eq_ignore_ascii_case("mpq") {
|
||||
let mut archive = match Archive::open(&input) {
|
||||
Ok(archive) => archive,
|
||||
Err(_) => continue,
|
||||
};
|
||||
match archive.read_file("(listfile)") {
|
||||
Ok(data) => data,
|
||||
Err(_) => continue,
|
||||
}
|
||||
} else if extension.eq_ignore_ascii_case("txt") || extension.eq_ignore_ascii_case("csv") {
|
||||
fs::read(&input)?
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
|
||||
for name in parse_listfile(&data)? {
|
||||
names.entry(name.to_ascii_lowercase()).or_insert(name);
|
||||
}
|
||||
sources += 1;
|
||||
}
|
||||
|
||||
if let Some(parent) = output.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut contents = names.into_values().collect::<Vec<_>>().join("\r\n");
|
||||
contents.push_str("\r\n");
|
||||
fs::write(output, contents)?;
|
||||
println!(
|
||||
"Wrote {} unique paths from {sources} listfiles to {}",
|
||||
parse_listfile(&fs::read(output)?)?.len(),
|
||||
output.display()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
use std::env;
|
||||
use std::error::Error;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use walkdir::WalkDir;
|
||||
use wow_mpq::compression::flags;
|
||||
use wow_mpq::{ArchiveBuilder, FormatVersion, ListfileOption};
|
||||
|
||||
fn collect_files(input_dir: &Path) -> Result<Vec<(PathBuf, String)>, Box<dyn Error>> {
|
||||
let root = input_dir.canonicalize()?;
|
||||
let mut files = Vec::new();
|
||||
|
||||
for entry in WalkDir::new(&root) {
|
||||
let entry = entry?;
|
||||
if !entry.file_type().is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let source = entry.path().canonicalize()?;
|
||||
let archive_path = source
|
||||
.strip_prefix(&root)?
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/");
|
||||
files.push((source, archive_path));
|
||||
}
|
||||
|
||||
files.sort_by(|left, right| {
|
||||
left.1
|
||||
.to_ascii_lowercase()
|
||||
.cmp(&right.1.to_ascii_lowercase())
|
||||
.then_with(|| left.1.cmp(&right.1))
|
||||
});
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn pack_directory(input_dir: &Path, output_path: &Path) -> Result<(), Box<dyn Error>> {
|
||||
if input_dir.join("__mpqmeta__").exists() {
|
||||
return Err(format!(
|
||||
"{} contains path-unresolved MPQ blocks and cannot be rebuilt safely",
|
||||
input_dir.display()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
let files = collect_files(input_dir)?;
|
||||
if files.is_empty() {
|
||||
return Err(format!("input directory contains no files: {}", input_dir.display()).into());
|
||||
}
|
||||
|
||||
if let Some(parent) = output_path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
let mut builder = ArchiveBuilder::new()
|
||||
.version(FormatVersion::V2)
|
||||
.block_size(7)
|
||||
.default_compression(flags::ZLIB)
|
||||
.listfile_option(ListfileOption::Generate);
|
||||
|
||||
let total = files.len();
|
||||
for (index, (source, archive_path)) in files.into_iter().enumerate() {
|
||||
builder = builder.add_file(source, &archive_path);
|
||||
let packed = index + 1;
|
||||
if packed.is_multiple_of(1000) {
|
||||
println!("Queued {packed}/{total} files...");
|
||||
}
|
||||
}
|
||||
|
||||
builder.build(output_path)?;
|
||||
println!(
|
||||
"Packed {total} files from {} to {}",
|
||||
input_dir.display(),
|
||||
output_path.display()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() != 3 {
|
||||
eprintln!("Usage: mpqpack <input_dir> <archive.mpq>");
|
||||
std::process::exit(2);
|
||||
}
|
||||
|
||||
pack_directory(Path::new(&args[1]), Path::new(&args[2]))
|
||||
}
|
||||
Reference in New Issue
Block a user