Files
felwell-core/dep/CascLib/src/CascCommon.h
T

465 lines
22 KiB
C

/*****************************************************************************/
/* CascCommon.h Copyright (c) Ladislav Zezula 2014 */
/*---------------------------------------------------------------------------*/
/* Common functions for CascLib */
/*---------------------------------------------------------------------------*/
/* Date Ver Who Comment */
/* -------- ---- --- ------- */
/* 29.04.14 1.00 Lad The first version of CascCommon.h */
/*****************************************************************************/
#ifndef __CASCCOMMON_H__
#define __CASCCOMMON_H__
//-----------------------------------------------------------------------------
// Compression support
// Include functions from zlib
#ifndef __SYS_ZLIB
#include "zlib/zlib.h"
#else
#include <zlib.h>
#endif
#include "CascPort.h"
#include "common/Common.h"
#include "common/Array.h"
#include "common/Map.h"
#include "common/FileTree.h"
#include "common/FileStream.h"
#include "common/Directory.h"
#include "common/ListFile.h"
#include "common/Csv.h"
#include "common/RootHandler.h"
// Headers from Alexander Peslyak's MD5 implementation
#include "md5/md5.h"
// For HashStringJenkins
#include "jenkins/lookup.h"
// On-disk CASC structures
#include "CascStructs.h"
//-----------------------------------------------------------------------------
// CascLib private defines
#ifdef _DEBUG
#define BREAK_ON_XKEY3(CKey, v0, v1, v2) if(CKey[0] == v0 && CKey[1] == v1 && CKey[2] == v2) { __debugbreak(); }
#define BREAKIF(condition) if(condition) { __debugbreak(); }
#else
#define BREAK_ON_XKEY3(CKey, v0, v1, v2) { /* NOTHING */ }
#define BREAKIF(condition) { /* NOTHING */ }
#endif
//-----------------------------------------------------------------------------
// In-memory structures
typedef enum _CBLD_TYPE
{
CascBuildNone = 0, // No build type found
CascBuildInfo, // .build.info
CascBuildDb, // .build.db (older storages)
CascVersionsDb // versions (downloaded online)
} CBLD_TYPE, *PCBLD_TYPE;
// Tag file entry, loaded from the DOWNLOAD file
typedef struct _CASC_TAG_ENTRY
{
USHORT HashType; // Hash type
char TagName[1]; // Tag name. Variable length.
} CASC_TAG_ENTRY, *PCASC_TAG_ENTRY;
// Normalized header of the index files.
// Both version 1 and version 2 are converted to this structure
typedef struct _CASC_INDEX_HEADER
{
USHORT IndexVersion; // 5 for index v 1.0, 7 for index version 2.0
BYTE BucketIndex; // Should be the same as the first byte of the hex filename.
BYTE StorageOffsetLength; // Length, in bytes, of the StorageOffset field in the EKey entry
BYTE EncodedSizeLength; // Length, in bytes, of the EncodedSize in the EKey entry
BYTE EKeyLength; // Length, in bytes, of the (trimmed) EKey in the EKey entry
BYTE FileOffsetBits; // Number of bits of the archive file offset in StorageOffset field. Rest is data segment index
BYTE Alignment;
ULONGLONG SegmentSize; // Size of one data segment (aka data.### file)
size_t HeaderLength; // Length of the on-disk header structure, in bytes
size_t HeaderPadding; // Length of padding after the header
size_t EntryLength; // Length of the on-disk EKey entry structure, in bytes
size_t EKeyCount; // Number of EKey entries. Only supplied for index files version 1.
} CASC_INDEX_HEADER, *PCASC_INDEX_HEADER;
// Normalized footer of the archive index files (md5.index)
typedef struct _CASC_ARCINDEX_FOOTER
{
BYTE TocHash[MD5_HASH_SIZE]; // Hash of the structure
BYTE FooterHash[MD5_HASH_SIZE]; // MD5 hash of the footer, possibly shortened
BYTE Version; // Version of the index footer
BYTE OffsetBytes; // Length, in bytes, of the file offset field
BYTE SizeBytes; // Length, in bytes, of the file size field
BYTE EKeyBytes; // Length, in bytes, of the EKey field
BYTE FooterHashBytes; // Length, in bytes, of the hash and checksum
BYTE Alignment[3];
size_t ElementCount; // total number of elements in the index file
size_t PageLength; // Length, in bytes, of the index page
size_t ItemLength; // Length, in bytes, of the single item
size_t FooterLength; // Length, in bytes, of the index footer structure
} CASC_ARCINDEX_FOOTER, *PCASC_ARCINDEX_FOOTER;
// Normalized structure for archive index entry
typedef struct _CASC_ARCINDEX_ENTRY
{
BYTE IndexHash[MD5_HASH_SIZE];
BYTE EKey[MD5_HASH_SIZE];
DWORD ArchiveOffset;
DWORD EncodedSize;
} CASC_ARCINDEX_ENTRY, *PCASC_ARCINDEX_ENTRY;
// Normalized header of the ENCODING file
typedef struct _CASC_ENCODING_HEADER
{
USHORT Magic; // FILE_MAGIC_ENCODING ('EN')
USHORT Version; // Expected to be 1 by CascLib
BYTE CKeyLength; // The content key length in ENCODING file. Usually 0x10
BYTE EKeyLength; // The encoded key length in ENCODING file. Usually 0x10
DWORD CKeyPageSize; // Size of the CKey page in bytes
DWORD EKeyPageSize; // Size of the CKey page in bytes
DWORD CKeyPageCount; // Number of CKey pages in the page table
DWORD EKeyPageCount; // Number of EKey pages in the page table
DWORD ESpecBlockSize; // Size of the ESpec string block, in bytes
} CASC_ENCODING_HEADER, *PCASC_ENCODING_HEADER;
typedef struct _CASC_DOWNLOAD_HEADER
{
USHORT Magic; // FILE_MAGIC_DOWNLOAD ('DL')
USHORT Version; // Version
USHORT EKeyLength; // Length of the EKey
USHORT EntryHasChecksum; // If nonzero, then the entry has checksum
DWORD EntryCount;
DWORD TagCount;
USHORT FlagByteSize;
USHORT BasePriority;
size_t HeaderLength; // Length of the on-disk header, in bytes
size_t EntryLength; // Length of the on-disk entry, in bytes
} CASC_DOWNLOAD_HEADER, *PCASC_DOWNLOAD_HEADER;
typedef struct _CASC_DOWNLOAD_ENTRY
{
BYTE EKey[MD5_HASH_SIZE];
ULONGLONG EncodedSize;
DWORD Checksum;
DWORD Flags;
BYTE Priority;
} CASC_DOWNLOAD_ENTRY, *PCASC_DOWNLOAD_ENTRY;
// Capturable tag structure for loading from DOWNLOAD manifest
typedef struct _CASC_TAG_ENTRY1
{
const char * szTagName; // Tag name
LPBYTE Bitmap; // Bitmap
size_t BitmapLength; // Length of the bitmap, in bytes
size_t NameLength; // Length of the tag name, in bytes, not including '\0'
size_t TagLength; // Length of the on-disk tag, in bytes
DWORD TagValue; // Tag value
} CASC_TAG_ENTRY1, *PCASC_TAG_ENTRY1;
// Tag structure for storing in arrays
typedef struct _CASC_TAG_ENTRY2
{
size_t NameLength; // Length of the on-disk tag, in bytes
DWORD TagValue; // Tag value
char szTagName[0x08]; // Tag string. This member can be longer than declared. Aligned to 8 bytes.
} CASC_TAG_ENTRY2, *PCASC_TAG_ENTRY2;
typedef struct _CASC_INSTALL_HEADER
{
USHORT Magic; // FILE_MAGIC_DOWNLOAD ('DL')
BYTE Version; // Version
BYTE EKeyLength; // Length of the EKey
DWORD EntryCount;
DWORD TagCount;
size_t HeaderLength; // Length of the on-disk header, in bytes
} CASC_INSTALL_HEADER, *PCASC_INSTALL_HEADER;
typedef struct _CASC_FILE_FRAME
{
CONTENT_KEY FrameHash; // MD5 hash of the file frame
DWORD DataFileOffset; // Offset in the data file (data.###)
DWORD FileOffset; // File offset of this frame
DWORD EncodedSize; // Encoded size of the frame
DWORD ContentSize; // Content size of the frame
} CASC_FILE_FRAME, *PCASC_FILE_FRAME;
//-----------------------------------------------------------------------------
// Structures for CASC storage and CASC file
struct TCascStorage
{
TCascStorage();
~TCascStorage();
TCascStorage * AddRef();
TCascStorage * Release();
static TCascStorage * IsValid(HANDLE hStorage);
PFNPROGRESSCALLBACK PfnCallback; // Callback to be called during CascOpen(Online)Storage
void * PtrCallbackParam;
const TCHAR * szIndexFormat; // Format of the index file name
const char * szClassName; // "TCascStorage"
const char * szProductName; // String representation of the product name
TCHAR * szRootPath; // Path where the build file is
TCHAR * szDataPath; // This is the directory where data files are
TCHAR * szIndexPath; // This is the directory where index files are
TCHAR * szBuildFile; // Build file name (.build.info or .build.db)
TCHAR * szCdnServers; // Multi-SZ list of CDN servers
TCHAR * szCdnPath; // Remote CDN sub path for the product
TCHAR * szCodeName; // Product code name. Only for online storages
char * szRegion; // Product region. Only when "versions" is used as storage root file
CASC_PRODUCT Product; // Product enum value (see CASC_PRODUCT)
DWORD dwBuildNumber; // Product build number
DWORD dwRefCount; // Number of references
DWORD dwDefaultLocale; // Default locale, read from ".build.info"
DWORD dwFeatures; // List of CASC features. See CASC_FEATURE_XXX
bool bAllowOrphans; // If TRUE, then orphaned items are allowed
CBLD_TYPE BuildFileType; // Type of the build file
QUERY_KEY CdnConfigKey; // Currently selected CDN config file. Points to "config\%02X\%02X\%s
QUERY_KEY CdnBuildKey; // Currently selected CDN build file. Points to "config\%02X\%02X\%s
QUERY_KEY ArchiveGroup; // Key array of the "archive-group"
QUERY_KEY ArchivesKey; // Key array of the "archives"
QUERY_KEY PatchArchivesKey; // Key array of the "patch-archives"
QUERY_KEY PatchArchivesGroup; // Key array of the "patch-archive-group"
QUERY_KEY BuildFiles; // List of supported build files
TFileStream * DataFiles[CASC_MAX_DATA_FILES]; // Array of open data files
CASC_CKEY_ENTRY EncodingCKey; // Information about ENCODING file
CASC_CKEY_ENTRY DownloadCKey; // Information about DOWNLOAD file
CASC_CKEY_ENTRY InstallCKey; // Information about INSTALL file
CASC_CKEY_ENTRY PatchFile; // Information about PATCH file
CASC_CKEY_ENTRY RootFile; // Information about ROOT file
CASC_CKEY_ENTRY SizeFile; // Information about SIZE file
CASC_CKEY_ENTRY VfsRoot; // The main VFS root file
CASC_ARRAY VfsRootList; // List of CASC_EKEY_ENTRY for each TVFS sub-root
TRootHandler * pRootHandler; // Common handler for various ROOT file formats
CASC_ARRAY ArcIndexArray; // Array of CASC_ARCINDEX_ENTRY, loaded from archive indexes
CASC_ARRAY IndexArray; // Array of CASC_CKEY_ENTRY loaded from local index files
CASC_ARRAY CKeyArray; // Array of CASC_CKEY_ENTRY, one entry for each physical file
CASC_ARRAY TagsArray; // Array of CASC_DOWNLOAD_TAG2
CASC_MAP ArcIndexMap; // Map of EKey -> CASC_ARCINDEX_ENTRY
CASC_MAP CKeyMap; // Map of CKey -> CASC_CKEY_ENTRY
CASC_MAP EKeyMap; // Map of EKey -> CASC_EKEY_ENTRY
size_t LocalFiles; // Number of files that are present locally
size_t TotalFiles; // Total number of files in the storage, some may not be present locally
size_t EKeyEntries; // Number of CKeyEntry-ies loaded from text build file
size_t OrphanItems; // Number of EKey entries in indexes that do not have their counterpart in ENCODING
size_t SkippedItems; // Number of EKey entries in indexes that were ignored due to insufficient capacity of CKeyArray
size_t EKeyLength; // EKey length from the index files
DWORD FileOffsetBits; // Nimber of bits in the storage offset which mean data segent offset
CASC_ARRAY ExtraKeysList; // List additional encryption keys
CASC_MAP EncryptionKeys; // Map of encryption keys
};
struct TCascFile
{
TCascFile(TCascStorage * ahs, PCASC_CKEY_ENTRY apCKeyEntry)
{
ULONGLONG StorageOffset;
ULONGLONG FileOffsMask;
// Reference the storage handle
hs = ahs->AddRef();
szClassName = "TCascFile";
// Init class variables
pCKeyEntry = apCKeyEntry;
pFrames = NULL;
pbFileCache = NULL;
cbFileCache = 0;
// Supply the sizes
StorageOffset = pCKeyEntry->StorageOffset;
FileOffsMask = ((ULONGLONG)1 << hs->FileOffsetBits) - 1;
ArchiveIndex = (DWORD)(StorageOffset >> hs->FileOffsetBits);
ArchiveOffset = (DWORD)(StorageOffset & FileOffsMask);
EncodedSize = pCKeyEntry->EncodedSize;
ContentSize = pCKeyEntry->ContentSize;
FilePointer = 0;
FrameCount = 0;
bVerifyIntegrity = false;
bDownloadFileIf = false;
bLocalFileStream = false;
}
~TCascFile()
{
// Close (dereference) the archive handle
hs = hs->Release();
szClassName = NULL;
// Free the file cache and frame array
CASC_FREE(pbFileCache);
CASC_FREE(pFrames);
// If we are supposed to close the file stream, do it
if (pStream && bLocalFileStream)
FileStream_Close(pStream);
}
static TCascFile * IsValid(HANDLE hFile)
{
TCascFile * hf = (TCascFile *)hFile;
return (hf != INVALID_HANDLE_VALUE &&
hf != NULL &&
hf->hs != NULL &&
hf->szClassName != NULL &&
hf->pCKeyEntry != NULL) ? hf : NULL;
}
TCascStorage * hs; // Pointer to storage structure
TFileStream * pStream; // An open data stream
const char * szClassName; // "TCascFile"
PCASC_CKEY_ENTRY pCKeyEntry;
PCASC_FILE_FRAME pFrames; // Array of file frames
DWORD ArchiveIndex; // Index of the archive (data.###)
DWORD ArchiveOffset; // Offset in the archive (data.###)
DWORD FilePointer; // Current file pointer
DWORD EncodedSize; // Encoded size. This is the size of encoded header, all file frame headers and all file frames
DWORD ContentSize; // Content size. This is the size of the file content, aka the file size
DWORD FrameCount; // Number of the file frames
DWORD bVerifyIntegrity:1; // If true, then the data are validated more strictly when read
DWORD bDownloadFileIf:1; // If true, then the data will be downloaded from the online storage if missing
DWORD bLocalFileStream:1; // If true, then the file stream is a local file
DWORD bOvercomeEncrypted:1; // If true, then CascReadFile will fill the part that is encrypted (and key was not found) with zeros
LPBYTE pbFileCache; // Pointer to file cache
DWORD cbFileCache; // Size of the file cache
};
struct TCascSearch
{
TCascSearch(TCascStorage * ahs, LPCTSTR aszListFile, const char * aszMask)
{
// Init the class
szClassName = "TCascSearch";
hs = ahs->AddRef();
// Init provider-specific data
pCache = NULL;
nFileIndex = 0;
nSearchState = 0;
bListFileUsed = false;
// Allocate mask
szListFile = CascNewStr(aszListFile);
szMask = CascNewStr((aszMask != NULL) ? aszMask : "*");
}
~TCascSearch()
{
// Dereference the CASC storage
hs = hs->Release();
szClassName = NULL;
// Free the rest of the members
CASC_FREE(szMask);
CASC_FREE(szListFile);
CASC_FREE(pCache);
}
static TCascSearch * IsValid(HANDLE hFind)
{
TCascSearch * pSearch = (TCascSearch *)hFind;
return (hFind != INVALID_HANDLE_VALUE &&
hFind != NULL &&
pSearch->szClassName != NULL &&
!strcmp(pSearch->szClassName, "TCascSearch") &&
pSearch->szMask != NULL) ? pSearch : NULL;
}
TCascStorage * hs; // Pointer to the storage handle
const char * szClassName; // Contains "TCascSearch"
TCHAR * szListFile; // Name of the listfile
void * pCache; // Listfile cache
char * szMask; // Search mask
// Provider-specific data
size_t nFileIndex; // Root-specific search context
DWORD nSearchState:8; // The current search state (0 = listfile, 1 = nameless, 2 = done)
DWORD bListFileUsed:1; // TRUE: The listfile has already been loaded
};
//-----------------------------------------------------------------------------
// Common functions (CascCommon.cpp)
bool OpenFileByCKeyEntry(TCascStorage * hs, PCASC_CKEY_ENTRY pCKeyEntry, DWORD dwOpenFlags, HANDLE * PtrFileHandle);
LPBYTE LoadInternalFileToMemory(TCascStorage * hs, PCASC_CKEY_ENTRY pCKeyEntry, DWORD * pcbFileData);
LPBYTE LoadFileToMemory(const TCHAR * szFileName, DWORD * pcbFileData);
void FreeCascBlob(PQUERY_KEY pQueryKey);
//-----------------------------------------------------------------------------
// Text file parsing (CascFiles.cpp)
int DownloadFileFromCDN(TCascStorage * hs, const TCHAR * szSubDir, LPBYTE pbEKey, const TCHAR * szExtension, TCHAR * szOutLocalPath, size_t cchOutLocalPath);
int CheckGameDirectory(TCascStorage * hs, TCHAR * szDirectory);
int LoadCdnsInfo(TCascStorage * hs);
int LoadBuildInfo(TCascStorage * hs);
int LoadCdnConfigFile(TCascStorage * hs);
int LoadCdnBuildFile(TCascStorage * hs);
//-----------------------------------------------------------------------------
// Internal file functions
void * ProbeOutputBuffer(void * pvBuffer, size_t cbLength, size_t cbMinLength, size_t * pcbLengthNeeded);
PCASC_CKEY_ENTRY FindCKeyEntry_CKey(TCascStorage * hs, LPBYTE pbCKey, PDWORD PtrIndex = NULL);
PCASC_CKEY_ENTRY FindCKeyEntry_EKey(TCascStorage * hs, LPBYTE pbEKey, PDWORD PtrIndex = NULL);
size_t GetTagBitmapLength(LPBYTE pbFilePtr, LPBYTE pbFileEnd, DWORD EntryCount);
int CascDecompress(LPBYTE pvOutBuffer, PDWORD pcbOutBuffer, LPBYTE pvInBuffer, DWORD cbInBuffer);
int CascDirectCopy(LPBYTE pbOutBuffer, PDWORD pcbOutBuffer, LPBYTE pbInBuffer, DWORD cbInBuffer);
int CascLoadEncryptionKeys(TCascStorage * hs);
int CascDecrypt(TCascStorage * hs, LPBYTE pbOutBuffer, PDWORD pcbOutBuffer, LPBYTE pbInBuffer, DWORD cbInBuffer, DWORD dwFrameIndex);
//-----------------------------------------------------------------------------
// Support for index files
int LoadIndexFiles(TCascStorage * hs);
//-----------------------------------------------------------------------------
// Support for ROOT file
int RootHandler_CreateMNDX(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateTVFS(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateDiablo3(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateWoW(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile, DWORD dwLocaleMask);
int RootHandler_CreateOverwatch(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateStarcraft1(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
int RootHandler_CreateInstall(TCascStorage * hs, LPBYTE pbRootFile, DWORD cbRootFile);
//-----------------------------------------------------------------------------
// Dumpers (CascDumpData.cpp)
#ifdef _DEBUG
void CascDumpFile(HANDLE hFile, const char * szDumpFile = NULL);
void CascDumpStorage(HANDLE hStorage, const char * szDumpFile = NULL);
#endif
#endif // __CASCCOMMON_H__